{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torchvision import transforms\n",
    "from torchvision.utils import make_grid\n",
    "\n",
    "import math\n",
    "import random\n",
    "\n",
    "from PIL import Image, ImageOps, ImageEnhance\n",
    "import numbers\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training samples: 42000\n",
      "Number of training pixels: 784\n",
      "Number of classes: 10\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('../datasets/digit-recognizer/train.csv')\n",
    "\n",
    "n_train = len(train_df)\n",
    "n_pixels = len(train_df.columns) - 1\n",
    "n_class = len(set(train_df['label']))\n",
    "\n",
    "print('Number of training samples: {0}'.format(n_train))\n",
    "print('Number of training pixels: {0}'.format(n_pixels))\n",
    "print('Number of classes: {0}'.format(n_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(train_df['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train samples: 28000\n",
      "Number of test pixels: 784\n"
     ]
    }
   ],
   "source": [
    "test_df = pd.read_csv('../datasets/digit-recognizer/test.csv')\n",
    "\n",
    "n_test = len(test_df)\n",
    "n_pixels = len(test_df.columns)\n",
    "\n",
    "print('Number of train samples: {0}'.format(n_test))\n",
    "print('Number of test pixels: {0}'.format(n_pixels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5, 3, 3, 3, 5, 8, 5, 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABKAAAACuCAYAAAABHHXfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAaBklEQVR4nO3dfbjX8/0H8E9dkbtUZBQqc1OxLnOTnEvF3EzIXEi5WZGoKRmatcQyY7Yr3QiTiC1zs+5sbsLltrLSxsXCZHM3hRByk10p9ftrv12f7+ud8+10Pud0znk8/ns/r/fn831t59v3+zkvn/P6NFq3bt26DAAAAAAK0ri2CwAAAACgftOAAgAAAKBQGlAAAAAAFEoDCgAAAIBCaUABAAAAUCgNKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIVqUu7GRo0aFVkHAAAAAHXQunXrKt3jDigAAAAACqUBBQAAAEChNKAAAAAAKJQGFAAAAACF0oACAAAAoFAaUAAAAAAUSgMKAAAAgEJpQAEAAABQKA0oAAAAAAqlAQUAAABAoTSgAAAAACiUBhQAAAAAhdKAAgAAAKBQGlAAAAAAFEoDCgAAAIBCaUABAAAAUCgNKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIXSgAIAAACgUBpQAAAAABRKAwoAAACAQjWp7QIAAAAAinbooYeG7Pzzzw/ZSSedVKXzN24c7/F58MEHQ9arV68qnb+ucwcUAAAAAIXSgAIAAACgUBpQAAAAABRKAwoAAACAQhlCDgXr06dPyCZOnBiy0uF0M2fODHtmz55dfYVRJ+y7774h69GjR8gOPPDAkHXq1ClkXbp0ya3XrVsX9jRq1ChkTz31VMhSwxNXrlwZMqBuGDRoUMgmTZoUstRnxLx580LWrVu3kE2dOjW3bteuXdjTsWPHkO24444hS9ljjz1y6zfeeKOs4wA21ogRI0LWvn37kJ133nk1UA3/VfozmDVrVtjTokWLkKWukcuxdu3aajtXfeQOKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIUyhLxAFRUVIevdu3fILr744iqdf8GCBSGbMWNGyMaNG1el81M9ttpqq5Bts802IRswYMA3rrMsyx5//PGQ9e/fP2TvvffehpTIJuS6667LrYcNGxb2lDs4PLXvb3/7W279wQcfhD2lg8qzLD34PJU99NBDIaMYQ4cODdl2221XpXMdccQRIdt5551DtnDhwpC9+uqrIVuxYkXIrr/++irVRs1J/dtPfY6kskMOOaSsff369atSbeUOcL388stz64EDB4Y9qQGxFKdJk/yvG6NHjw57jj322JDtv//+ZZ1/6dKlIXv++edz67lz54Y9U6ZMCdknn3xS1mtClmXZaaedlltfccUVYc/48eNrqBrWZ7PNNsutmzdvXkuVkGXugAIAAACgYBpQAAAAABRKAwoAAACAQjVaV+Yf1afmi/A/qTlOY8eOrYVKIj+7uqF0PsqZZ54Z9qRmR61evTpkp556asjuvffejaiOmnLeeefl1h07dgx7li9fHrJnn302ZKmZFytXrqy0hgsuuCBkEyZMCNlxxx0XMjOg/qdr164hO+yww0KWmrtz/PHHV3r+0pkGG6L0eyF1KbBs2bKQpd4/u+++e1mvOW3atNz6N7/5TdhTOreFYpXODEtdt6TmDNYlqVlmqfc21aNVq1Yhmz59em596KGHhj0fffRRyBYtWhSyp556KmSlM6ayLMuOPvro3Lpdu3ZhT+ozdOTIkSG75ZZbQkbDs9dee4XsiSeeyK3ffffdsKf0vZhlZo3VtD333DO3fuWVV8KeefPmhWzmzJkhS33GXXbZZbl16nfv2bNnh6yca726ppzWkjugAAAAACiUBhQAAAAAhdKAAgAAAKBQGlAAAAAAFCpO7aNS8+fPD1lFRUVZx5YOYsyyLJsxY8ZG1/RfS5YsqbZzUbOGDRuWW48ZMybsueKKK0LWr1+/kP3hD38IWe/evUNmYPSm56abbqrtErLu3buHLDVUMDXEsaH605/+FLKePXuGbPPNNy/rfC+//HJu/dhjj5V13OLFi0N25513lnVsqbVr14Ys9T7o1KlTyFJD8fv06ZNbr1q1KuxJPXyB4vTt2ze3Lnfg+OTJk0OWejjCgAEDqlTXq6++GrLU4PBPP/00ZG3bts2tP/vssyrVQNXcf//9ITvooINy60mTJoU9qeHfqZ9vuUaPHp1bN2vWLOy59tprQ1b6QJgsy7Kvv/46ZLfddluVa2PTl7q2njhxYshKB9mnvvcNHK99//rXv3Lr1IMLNkaLFi1y69Lf6chzBxQAAAAAhdKAAgAAAKBQGlAAAAAAFEoDCgAAAIBCGUJehmnTpuXW5Q4cHz58eMjGjRtXLTVR/7399tshO/vss0OWej926NAhZAcffHDIDCEn5eSTTw5Z6mEJb731Vg1UUzfccccdIevVq1fIbr311pBdeeWVIVuxYkVu/cUXX1S9uGp07LHHhuzSSy+ttnNRs/bff/8qHffII4+ELDWI//LLL6/S+akbjjnmmJCl3lNPPvlkbj1kyJDCalqfzz//PGSDBw8OWeqhDUOHDg2ZIeR10+GHHx6y1Pd369atQ5Z6D51wwgm59UsvvbQR1VEXtG/fPmSpofWsnzugAAAAACiUBhQAAAAAhdKAAgAAAKBQGlAAAAAAFMoQ8hIXX3xxyE455ZRKj2vbtm3IlixZUi01ARQlNSR45cqVIRs/fnxNlFNnzZw5M2S77rpryN5///2QrV27tpCaNsSOO+4YstSA3pEjR4asadOmZb3Gl19+mVunHqrApmfZsmUh21SG4lO7Ug8gaNSoUchS19abqrlz54bshRdeqPlC+EYtW7YMWep3sXPOOSe3HjRoUNjTpEn8dfjhhx8O2bnnnhuyd9555xvrpP5p06ZNyJo3b17pcR9++GER5dRJ7oACAAAAoFAaUAAAAAAUSgMKAAAAgEJpQAEAAABQKEPIS1x44YWV7pk+fXrIDBynujVuHPvDEydODFm5g45nzZpVPYVRpx155JG59YgRI8Ke1157LWQLFiworKb66r333qvtEtarZ8+euXVqGH1FRUVZ5/rss89Cdt9994WsdFD7/fffX9b5qV3XXHNNyB577LGQ7bfffiHr06dPyNq3b59bP/roo2HPbbfdtgEVUlu23XbbkK1ZsyZkixYtqolyqkXq2qv0AQrUrC5duoQsdT3ctWvXkH311Ve5der65rTTTgtZXXrPUrNGjRpVpeOuvvrqaq6k7nIHFAAAAACF0oACAAAAoFAaUAAAAAAUSgMKAAAAgEI16CHkqeHNqaxUaqgmbIgWLVqErHfv3rn1yJEjw57ddtstZMuWLQvZ97///ZC99NJLG1Ah9cF5550XsjFjxuTWW265Zdjzy1/+srCaqHk9evQIWelDCbbYYouyzvXpp5+GrF+/fiF74IEHyqyOmtK2bduQbb/99pUed9VVV4Xs8MMPD1mHDh1Ctnz58pB169Ytt95pp53Cnj322CNkixcvDtnUqVNDBhujb9++ISt9aEOWZdmZZ55ZE+WQZdkvfvGLkB100EEhSz0kavz48bn1M888U32FUe+VPjQjy7KsXbt2IWvUqFFu/eKLL4Y9qeunhsodUAAAAAAUSgMKAAAAgEJpQAEAAABQqAY9A6pcS5Ysqe0S2ATtt99+IRsxYkTIli5dGrLU366XzsVIefrpp0N2wQUXhMy8p/qtS5cuIXvwwQdD1qpVq5CV/p36z372s7Bn5syZG1Edm5rnn38+ZK+++mpuve+++5Z1rqZNm4YsdeyqVatC9uijj5b1Gmy8bbfdNmRz5swJWWouVKlmzZqF7IQTTqhaYQmpGWWp7L333gvZDjvsELKxY8dWT2FU6uOPPw5ZkybxV4uuXbvm1gsXLiyspg3Rv3//kN10000h+/GPf1wT5ZBl2ZAhQ0KWmsGVmveUmt8F5UrNe7r33ntD1rFjx5CtW7cut540aVLYk5qJ2FC5AwoAAACAQmlAAQAAAFAoDSgAAAAACqUBBQAAAEChGq0rnZq1vo0lg2vrq3L+71iwYEHIdtlll5DtuuuuVaohNfT8kEMOKWsfNefvf/97yDp37lxt51+zZk3Itthii5CtXbu22l6TumH48OEhGzNmTMhSn2eln+VTp04Ne84666yqF0edUDpEc++99w57zjnnnJAdcMABIUsNgv7qq69CNn/+/Nw69Z5NKR2YnmVZ9sYbb5R1bEO1/fbbh+yDDz6ohUqKtWzZspCVDiEfN25cTZXT4Hzve98L2f333x+yTz75JLceOHBg2FP6+ZBlWVZRURGyvfbaK2QHH3zwN9a5PqeeemrI/vnPf4Ys9aCF1DUaGyb1gIOJEyeG7JhjjglZ6gEKqe8dKNewYcNCNn78+LKO/fDDD3Pr1q1bV0tNdVE5vRR3QAEAAABQKA0oAAAAAAqlAQUAAABAoTSgAAAAACiUIeQlUgMPU4MRy1HukPByhpWnhmimBhFTc9q0aROyAQMGlLUvNUSzdJhn48axP/zwww+H7IwzzghZ6cBP6r/27duHbPDgwSEbMWJEpefq06dPyGbMmFGluqhfOnXqFLKWLVuWdeyJJ56YW3ft2jXs6datW8gmT54csl//+tche+utt8qqoyGoziHkL774YshuuOGGkKWGxVdVagB+6fsny7Js6623DlnpYPLUw0E+/vjjjaiOb3LUUUeF7JFHHqmFSvLefPPNkKUe4rL77ruHbNSoUSG75pprqqewBuzCCy8MWelDBLIsyxYuXBiyk046qazX+Pzzz3PrlStXllccDc6TTz4Zsu7du5d1bOl7OfUd2VAYQg4AAABArdOAAgAAAKBQGlAAAAAAFEoDCgAAAIBCGUJeBakBvamB4wsWLKjS+cv8kfiZ1DO9evXKre++++6wJzVwdfr06SEbOnRoyJYvX74R1VFflA5Z7NGjR9izaNGikO23336F1UTD1KJFi5C9/PLLIWvdunXIUgOFU8ODG6pyh5CXDhhPDU699957Q/bRRx9tRHVVc+ONN4asf//+Idtqq61y68svvzzs+dWvflV9hTVgzZs3D9nEiRND1q9fv0rP9f7774esdID0+rzxxhshu+KKK3Lr5557LuxJDSFPPUxm/PjxIfvBD36QW6cGGJP37W9/O7eeN29e2LPTTjtV+fyp34tKH45w/vnnhz2PP/54lV+TuqH0eiP1vXbYYYeFLPUZMWXKlJANGjSoyrXVN4aQAwAAAFDrNKAAAAAAKJQGFAAAAACF0oACAAAAoFCGkG8CKioqcuv58+eHPakh523bti2sJmpf6VDyLMuy++67r6xj+/btG7LUsHIangsuuCC3njBhQtjz2muvhWyvvfYqqiT4f6n3Y+l7dn0aN/bf1P6rSZMmIevcuXPI3n777dy6NoaLb4zUd+Jxxx2XW6cewHH88ceH7K9//Wv1FVYPnX322SFLDXPfYYcdQlY68Pfmm28Oe+bOnRuy1ADglNWrV5e1rxypz5HLLrssZKUD8I8++uiw5/XXX6+2uuqapk2bhuyOO+7IrXv37l1T5fy///znPyGbMWNGyEaOHBmyd999t5CaKN6ee+6ZW7/yyithT6rXkXp4R+r3s9RDDhoqQ8gBAAAAqHUaUAAAAAAUSgMKAAAAgELFIQHUuIsuuqjSPUuXLq2BStiUzJ49O2R33XVXyE4//fSQnXLKKSF74IEHQpb6W3jqt9K/zU79rfacOXNqqhwasG233TZkqdk8KeZSfrM1a9aE7Pnnn6+FSor18ssvh6x0BlSrVq3CntScRDOg/mf//fcP2SWXXBKybbbZJmQ//elPQzZ27NjqKawGpOZOPfnkkyEbPXp0bp2aC3PddddVX2F1TMuWLUN28skn59aff/552DN58uSQXXvttWW9ZuvWrUN2xBFH5Nap37l++MMfhuyrr74K2ZAhQ0JWnfPHqB7HHHNMyFKfS+U499xzQ2be08ZzBxQAAAAAhdKAAgAAAKBQGlAAAAAAFEoDCgAAAIBCGUJew/r06ROy1MDoUqmBmdRvqUGYo0aNCtnRRx8dst69e4fszjvvDNmf//znKlbHf7Vv3z5kN954Y8huvvnmkN13331FlPSNtt5669w6Ncx58eLFNVUODchmm22WW++zzz5hz2677VbWuVLD86k/2rZtG7LUQzgOPPDAmiinwUl9h3Xo0CFkqYeg3HPPPYXUVJteeeWVkJV+BqWGbjdkH330Uch+8pOf5NaLFi0Kex5//PEqv+b7778fshdeeCG3fvTRR8Oeu+++O2QDBw4M2TPPPBOyKVOmbECF1IQJEyaEbPfdd6/SuUoH52dZlh155JEhmzVrVm49d+7cKr1eQ+EOKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIUyhLxAFRUVIfvjH/9Y6XHjxo0L2ZIlS6qlJuq2L774ImSrVq0q69jly5dXdzlk6QGFPXv2DFm7du1CtmDBgpB9+OGH1VPYegwePDi3Tg1znjRpUqE1UP+VDhzPsiwbMWJEbn3llVdW+fy33nprlY+ldvXq1Su37tatW9gzYMCAkLVq1aqwmhq6E088Mbc+6KCDwp6JEyeGbNq0aYXVtCnp0qVLyBo39t/wv8nq1atDNn78+FqoJC81+HzQoEEhmzdvXshS13uGkNec1ENz1qxZU23nT/2b7t+/f1nHDhs2rNJzvfTSSyEr/T7Msiz797//XdZr1mU+PQEAAAAolAYUAAAAAIXSgAIAAACgUBpQAAAAABTKEPJqsuuuu4Zs/vz5ZR1bOoh4+PDh1VITNa9NmzYhW7ZsWcjWrl1b6blSw/bOP//8sl7zueeeC9mLL75Y6Wuy4b788suQpX52e++9d8h++9vfhuzcc8/NrVesWFFWHTvvvHPIJkyYELLSYehTp04Ne1auXFnWa9LwbL/99iHbZZddQjZq1KiQnXLKKbl1agB+KrvttttClhoay/+MHDkyZJ9++mmlx6V+vt/5zndC1r1796oVlmXZdtttl1unBtZXp9Tg13KvzxqK7373u5XumT59esjKuZapD5o1a1bpntdff70GKqEIf/nLX0KWuo5r3bp1TZTDBkhdM1RV6vOsqudPnatTp04he/rpp0N26aWXhuyOO+6oUh2bKndAAQAAAFAoDSgAAAAACqUBBQAAAEChNKAAAAAAKJQh5GUoHTBeOkg1y7Js7NixZZ2rdOB4lmVZ3759q1YYm5zbb789ZL///e9Ddtddd4WsdDDr5MmTw56TTjopZKtXrw5Zalj5Z599FjI23qRJk0LWuXPnkA0ePDhkJ598cshKh4SPHj067EkNND/77LND1rFjx5DNmTMnt77wwgvDHuq/zTffPLdOvWfPOeeckPXs2TNkpe/Z9fn6669z63feeSfsufLKK0M2ZcqUss7P/3zrW98K2VVXXVULldS80qHjRx11VNjzwQcf1FQ5dcIRRxyRW7/wwgthT2pQc0MxZMiQkJUOHX/kkUdqqhxqQGr49IwZM2qhEmpK6lrjwAMPDFlqGP0OO+xQpddMnat58+ZVOldd4g4oAAAAAAqlAQUAAABAoTSgAAAAAChUg54BNW3atJAtWbIkZKUzn0pnQq1P6lypeU+pfdQfv/vd70J2yy23hKxx43w/uGnTpmHPl19+GbIzzjgjZAsXLtyACtkYqTkBl1xyScjatGkTshNOOCFkpX9vPnv27LJeM+Whhx4K2VlnnZVbr1ixoqxzUT1KZy9lWZZtscUWlR637777hmzp0qUhS80hOP3000NW+j124oknVlrD+qxduzZkqdl3pTMQb7311iq/Jg3P4sWLQ5aaSVY632nVqlWF1VRfbLnllrl16t90fZT67B0xYkTIunfvHrJLL700tzZXrG7YZZddQpaauZr63Hj22WcLqYmqS80xTc0/LZ25evXVV4c9N9xwQ8hSMy5TM5p69OiRW1933XVhzz/+8Y+Qpd57c+fODVl94w4oAAAAAAqlAQUAAABAoTSgAAAAACiUBhQAAAAAhWq0rsxpto0aNSq6lkKlBoe//fbb1Xb+4cOHh2zcuHHVdn7qhosuuihkpUOfsyzLOnfuXOm5Zs6cGbLRo0eHLDXUjrph4MCBISsdBr3PPvuEPXPmzAnZrFmzQpYaQr569eoNKZFqlhqYual+V6xZsyZkU6dODVnpcPEsy7IpU6YUUhPlSQ1YTf08S6WGLafstNNOIevQoUPIUtdZpe+h1157Ley55557Qpa6XC3nfxOVe+KJJ3Lr0oG6WZYe7Jv62W2qDjvssJCNGTMmZAcccEDIJk6cGLLS671yHw7ChuvVq1fIWrZsGbLUw16+973v5dYHH3xw2NOsWbOQpR4aNWPGjG+sEyjvs9AdUAAAAAAUSgMKAAAAgEJpQAEAAABQKA0oAAAAAArVYIaQp8yfPz9kFRUVISsdEJsaOA4AlenatWvISgcAZ1mWbbnlloXW8eabb+bWt99+e9hz9913h+z1118vrCagdgwdOjS3vv7668Oe5557LmQ///nPQ5Z6+EXRUoPDzzzzzNz6Rz/6Udizdu3akF188cUhmzx5csgMwAeIDCEHAAAAoNZpQAEAAABQKA0oAAAAAAqlAQUAAABAoRr0EHIAAAAANo4h5AAAAADUOg0oAAAAAAqlAQUAAABAoTSgAAAAACiUBhQAAAAAhdKAAgAAAKBQGlAAAAAAFEoDCgAAAIBCaUABAAAAUCgNKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIXSgAIAAACgUBpQAAAAABRKAwoAAACAQmlAAQAAAFAoDSgAAAAACqUBBQAAAEChNKAAAAAAKJQGFAAAAACF0oACAAAAoFAaUAAAAAAUqkm5G9etW1dkHQAAAADUU+6AAgAAAKBQGlAAAAAAFEoDCgAAAIBCaUABAAAAUCgNKAAAAAAKpQEFAAAAQKE0oAAAAAAolAYUAAAAAIXSgAIAAACgUP8HyAVec9+U6NoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1600x200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "random_sel = np.random.randint(n_train, size=8)\n",
    "grid = make_grid(torch.Tensor((train_df.iloc[random_sel, 1:].values/255.).reshape((-1,28,28))).unsqueeze(1), nrow=8)\n",
    "plt.rcParams['figure.figsize'] = (16, 2)\n",
    "plt.imshow(grid.numpy().transpose((1,2,0)))\n",
    "plt.axis('off')\n",
    "print(*list(train_df.iloc[random_sel, 0].values), sep = ', ')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAHGCAYAAAB3gE0oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAz5klEQVR4nO3deXhV9bm38TuEJEAgICiEyBSxMqiAgGCcCohJabC20BYRFUHkaAMV0+NAq0zWOh1BKwjiALSaorZqFVAIUaAIyKCpDJaionhEghMJEA0h2e8ffdmnKagQdrID6/5cF1e713r2Ws+zc6lfVn57rZhQKBRCkiRJCrha0W5AkiRJqgkMxpIkSRIGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBEDtaDdwLCsvL2f79u00aNCAmJiYaLcjSZKk/xAKhdi9ezcpKSnUqvXt14QNxkdh+/bttGzZMtptSJIk6Tt89NFHtGjR4ltrDMZHoUGDBsC/PuikpKQodyNJkqT/VFRURMuWLcO57dsYjI/CgeUTSUlJBmNJkqQa7HCWvfrlO0mSJAmDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAqB2tBuQDqXNrfOj3UKlfHB3ZrRbkCRJleQVY0mSJAmDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJgNrRbkCSJFWvNrfOj3YLlfLB3ZnRbkHHOa8YS5IkSRiMJUmSJMBgLEmSJAGuMZZUxVzLKEk6VnjFWJIkScJgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEkA1I52A5IkSaqcNrfOj3YLlfLB3ZnRbuGQDMZSFPkvNEmSag6XUkiSJEkYjCVJkiTApRTHJH/9LklVx3/HSsHlFWNJkiQJg7EkSZIEGIwlSZIkwGAsSZIkAX75TpIiwi9sSdKxr8YH47vvvpuxY8dyww038MADDwDw9ddf86tf/Yq5c+dSUlJCRkYGDz/8MM2aNQu/b9u2bVx//fW89tpr1K9fn6FDh3LXXXdRu/b/jbxkyRKys7PZuHEjLVu25LbbbuPqq6+u5gklSVJV8C+sOlI1einFmjVreOSRR+jUqVOF7TfeeCMvvfQSzz77LEuXLmX79u0MGDAgvL+srIzMzEz27dvHihUrmDNnDrNnz2bcuHHhmq1bt5KZmUnv3r3Jz89nzJgxjBgxgoULF1bbfJIkSao5amww3rNnD0OGDOHRRx/lhBNOCG8vLCzk8ccfZ/LkyfTp04du3boxa9YsVqxYwapVqwBYtGgRmzZt4sknn6RLly7069ePO+64g2nTprFv3z4AZsyYQWpqKvfffz8dOnRg1KhR/PSnP2XKlClRmVeSJEnRVWODcVZWFpmZmfTt27fC9nXr1lFaWlphe/v27WnVqhUrV64EYOXKlZx55pkVllZkZGRQVFTExo0bwzX/eeyMjIzwMSRJkhQsNXKN8dy5c3nzzTdZs2bNQft27NhBfHw8jRo1qrC9WbNm7NixI1zz76H4wP4D+76tpqioiK+++oq6desedO6SkhJKSkrCr4uKigAoLS2ltLT0CKesvITYULWdK5KO5DMKwowQjDmDMCMEZ84gCMLPMggzQjDmDMKM1XmuGheMP/roI2644QZyc3OpU6dOtNup4K677mLixIkHbV+0aBH16tWrtj7u7VFtp4qoBQsWHHZtEGaEYMwZhBkhOHMGQRB+lkGYEYIxZxBmPFrFxcWHXVvjgvG6devYuXMnXbt2DW8rKytj2bJlTJ06lYULF7Jv3z527dpV4apxQUEBycnJACQnJ7N69eoKxy0oKAjvO/C/B7b9e01SUtIhrxYDjB07luzs7PDroqIiWrZsSXp6OklJSZUf+gidMeHY/ILghgkZh10bhBkhGHMGYUYIzpxBEISfZRBmhGDMGYQZj9aB3/AfjhoXjC+66CLWr19fYduwYcNo3749t9xyCy1btiQuLo68vDwGDhwIwObNm9m2bRtpaWkApKWlceedd7Jz506aNm0KQG5uLklJSXTs2DFc859/W8nNzQ0f41ASEhJISEg4aHtcXBxxcXGVH/oIlZTFVNu5IulIPqMgzAjBmDMIM0Jw5gyCIPwsgzAjBGPOIMxYneeqccG4QYMGnHHGGRW2JSYm0qRJk/D2a665huzsbBo3bkxSUhKjR48mLS2Nc845B4D09HQ6duzIlVdeyb333suOHTu47bbbyMrKCgfb6667jqlTp3LzzTczfPhwXn31VZ555hnmzz8273koSZKko1PjgvHhmDJlCrVq1WLgwIEVHvBxQGxsLPPmzeP6668nLS2NxMREhg4dyqRJk8I1qampzJ8/nxtvvJEHH3yQFi1a8Nhjj5GR4a8VJUmSguiYCMZLliyp8LpOnTpMmzaNadOmfeN7Wrdu/Z0Lu3v16sVbb70ViRYlSZJ0jKux9zGWJEmSqpPBWJIkScJgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJOAYeSS0JCn62tw6P9otVMoHd2dGuwVJxwivGEuSJEkYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgTU0GA8ffp0OnXqRFJSEklJSaSlpfHyyy+H93/99ddkZWXRpEkT6tevz8CBAykoKKhwjG3btpGZmUm9evVo2rQpN910E/v3769Qs2TJErp27UpCQgKnnnoqs2fPro7xJEmSVAPVyGDcokUL7r77btatW8fatWvp06cPl156KRs3bgTgxhtv5KWXXuLZZ59l6dKlbN++nQEDBoTfX1ZWRmZmJvv27WPFihXMmTOH2bNnM27cuHDN1q1byczMpHfv3uTn5zNmzBhGjBjBwoULq31eSZIkRV/taDdwKJdcckmF13feeSfTp09n1apVtGjRgscff5ycnBz69OkDwKxZs+jQoQOrVq3inHPOYdGiRWzatInFixfTrFkzunTpwh133MEtt9zChAkTiI+PZ8aMGaSmpnL//fcD0KFDB5YvX86UKVPIyMio9pklSZIUXTUyGP+7srIynn32Wfbu3UtaWhrr1q2jtLSUvn37hmvat29Pq1atWLlyJeeccw4rV67kzDPPpFmzZuGajIwMrr/+ejZu3MhZZ53FypUrKxzjQM2YMWO+sZeSkhJKSkrCr4uKigAoLS2ltLQ0QhN/t4TYULWdK5KO5DMKwowQjDmDMCMEY84gzAjBmDMIM0Iw5gzCjNV5rphQKFQjP9H169eTlpbG119/Tf369cnJyeGHP/whOTk5DBs2rEJABejRowe9e/fmnnvuYeTIkXz44YcVlkUUFxeTmJjIggUL6NevH6eddhrDhg1j7Nix4ZoFCxaQmZlJcXExdevWPainCRMmMHHixIO25+TkUK9evQhOL0mSpEgoLi7m8ssvp7CwkKSkpG+trbFXjNu1a0d+fj6FhYX8+c9/ZujQoSxdujSqPY0dO5bs7Ozw66KiIlq2bEl6evp3ftCRdMaEY3Md9IYJh79EJQgzQjDmDMKMEIw5gzAjBGPOIMwIwZgzCDMerQO/4T8cNTYYx8fHc+qppwLQrVs31qxZw4MPPsigQYPYt28fu3btolGjRuH6goICkpOTAUhOTmb16tUVjnfgrhX/XvOfd7IoKCggKSnpkFeLARISEkhISDhoe1xcHHFxcZUbtBJKymKq7VyRdCSfURBmhGDMGYQZIRhzBmFGCMacQZgRgjFnEGasznPVyLtSHEp5eTklJSV069aNuLg48vLywvs2b97Mtm3bSEtLAyAtLY3169ezc+fOcE1ubi5JSUl07NgxXPPvxzhQc+AYkiRJCpYaecV47Nix9OvXj1atWrF7925ycnJYsmQJCxcupGHDhlxzzTVkZ2fTuHFjkpKSGD16NGlpaZxzzjkApKen07FjR6688kruvfdeduzYwW233UZWVlb4iu91113H1KlTufnmmxk+fDivvvoqzzzzDPPnz4/m6JIkSYqSGhmMd+7cyVVXXcUnn3xCw4YN6dSpEwsXLuTiiy8GYMqUKdSqVYuBAwdSUlJCRkYGDz/8cPj9sbGxzJs3j+uvv560tDQSExMZOnQokyZNCtekpqYyf/58brzxRh588EFatGjBY4895q3aJEmSAqpGBuPHH3/8W/fXqVOHadOmMW3atG+sad26NQsWLPjW4/Tq1Yu33nqrUj1KkiTp+HLMrDGWJEmSqpLBWJIkScJgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCjiIYDx8+nCeeeOI762bPns3w4cMrexpJkiSpWlQ6GM+ePZvly5d/Z93rr7/OnDlzKnsaSZIkqVpU+VKKsrIyatVyxYYkSZJqtipPrFu2bKFhw4ZVfRpJkiTpqBzRI6EnTZpU4XV+fv5B2w7Yv38/GzduZMWKFfTt27fyHUqSJEnV4IiC8YQJE4iJiSEUCgH/Csb5+fnf+p7ExETGjRtX6QYlSZKk6nBEwXjcuHHhYDxp0iS6dOnCpZdeesja+Ph4WrRoQUZGBk2bNo1Is5IkSVJVOeIrxgccCMbjx4+PdE+SJElStTuiYPzvysvLI9mHJEmSFFXeR02SJEniKK4YH7B9+3Zee+01Pv74Y77++utD1sTExHD77bcf7akkSZKkKnNUwTg7O5upU6dSVlYGEL5bxQEHvqhnMJYkSVJNV+lgPHnyZB544AFiYmLIyMigQ4cOJCUlRbI3SZIkqdpUOhg//vjj1K5dm0WLFtGrV68ItiRJkiRVv0p/+e69997j/PPPNxRLkiTpuFDpYNygQQOaN28eyV4kSZKkqKl0ML7gggv4+9//HsleJEmSpKipdDAeN24c7777Lo899lgk+5EkSZKiotJfvisqKiI7O5v/+q//YtGiRfTv359WrVpRq9ahs/aFF15Y6SYlSZKkqlbpYNyrV6/wfYr/8pe/8Je//OUba2NiYti/f39lTyVJkiRVuUoH4wsvvJCYmJhI9iJJkiRFTaWD8ZIlSyLYhiRJkhRdlf7ynSRJknQ8MRhLkiRJHMVSimXLlh1RvXelkCRJUk121HelOBzelUKSJEk1XcTvSlFeXs6HH37IRx99BEBaWhpxcXGV71CSJEmqBlV2V4q3336bq6++msTERBYsWFDZ00iSJEnVosq+fNepUyeee+45li9fzn333VdVp5EkSZIiokrvStGmTRvOPvts/vCHP1TlaSRJkqSjVuW3azvppJP44IMPqvo0kiRJ0lGp0mC8b98+1qxZQ7169aryNJIkSdJRq5JgvHfvXtauXcvAgQP56KOP6N27d1WcRpIkSYqYSt+VIjY29jtrQqEQjRo14re//W1lTyNJkiRVi0pfMQ6FQt/4p3bt2rRu3ZoRI0bw5ptv0q5du0j2LEmSJEVcpa8Yl5eXR7IPSZIkKaqq/K4UkiRJ0rHAYCxJkiRxFEspDvj888959NFHee211/j4448BOPnkk+nTpw8jRoygSZMmR92kJEmSVNWOKhgvWrSIwYMHs2vXLkKhUHj7pk2bWLx4Mffddx85OTmkp6cfdaOSJElSVap0MN6yZQsDBgyguLiYTp06MWzYMNq2bQvA+++/z+zZs8nPz2fAgAG89dZbfO9734tY05IkSVKkVToY33333RQXFzNhwgTGjRt30P5f/vKX3HHHHYwfP5577rmHxx577KgalSRJkqpSpb98l5eXR7t27Q4Zig+4/fbbadeuHYsXL67saSRJkqRqUelgvGPHDrp27fqddV27dmXHjh2VPY0kSZJULSodjBMTE9m5c+d31u3cuZPExMTKnkaSJEmqFpUOxl26dGHZsmWsX7/+G2vefvttli5dSpcuXSp7GkmSJKlaVDoYX3vttZSWltK3b18efvhh9uzZE963Z88epk6dysUXX0xZWRkjR46MSLOSJElSVan0XSkuu+wyXn75Zf74xz8yevRoRo8eHX6Yx+effw5AKBTiqquuYtCgQZHpVpIkSaoiR/VI6Dlz5vDwww+TmppKKBTis88+47PPPiMUCnHKKacwffp0Zs+eHaFWJUmSpKpz1I+Evu6667juuuv4+OOPKzwS+uSTTz7q5iRJkqTqckTBeM2aNXzyySd06NDhoCfZ/WcY3rJlC++88w4pKSl07949Mt1KkiRJVeSwg/Fnn33GRRddRIMGDcjPz//O+kaNGvGLX/yC4uJi3n//fRo1anQUbUqSJElV67DXGD/55JPs2bOHiRMnctJJJ31n/UknncSkSZPYtWsXTz755FE1KUmSJFW1ww7GCxYsIDExkaFDhx72wa+88krq16/PvHnzKtWcJEmSVF0OOxhv2LCBnj17EhcXd9gHj4uLo0ePHt/6EBBJkiSpJjjsYPzFF1+QnJx8xCdo1qxZ+L7GkiRJUk112ME4ISGBvXv3HvEJiouLSUhIOOL3SZIkSdXpsINxcnIyb7/99hGf4O23367UlWZJkiSpOh12MD733HP54IMPWLFixWEf/PXXX2fr1q2ce+65lWpOkiRJqi6HHYyHDBlCKBRi5MiRFBYWfmf9rl27GDlyJDExMQwePPiompQkSZKq2mEH4759+3LRRRexadMmunXrxosvvkgoFDqoLhQK8de//pXu3bvzj3/8g169epGenh7RpiVJkqRIO+xgDDB37lxOO+003n//fX7yk59w4okncvHFFzNkyBCGDBnCxRdfzIknnsiAAQN4//33adu2LU8//fQRN3XXXXdx9tln06BBA5o2bcqPf/xjNm/eXKHm66+/JisriyZNmlC/fn0GDhxIQUFBhZpt27aRmZlJvXr1aNq0KTfddBP79++vULNkyRK6du1KQkICp556KrNnzz7ifiVJknTsO6Jg3KRJE1avXs0VV1xBrVq1+PLLL8nLy2Pu3LnMnTuXvLw8vvzyS2JiYhgyZAirV6/mxBNPPOKmli5dSlZWFqtWrSI3N5fS0lLS09Mr3BXjxhtv5KWXXuLZZ59l6dKlbN++nQEDBoT3l5WVkZmZyb59+1ixYgVz5sxh9uzZjBs3LlyzdetWMjMz6d27N/n5+YwZM4YRI0awcOHCI+5ZkiRJx7baR/qGpKQk/vCHPzBx4kTmzZvH2rVr+fTTT4F/PQa6W7du9O/fn1NOOaXSTb3yyisVXs+ePZumTZuybt06LrzwQgoLC3n88cfJycmhT58+AMyaNYsOHTqwatUqzjnnHBYtWsSmTZtYvHgxzZo1o0uXLtxxxx3ccsstTJgwgfj4eGbMmEFqair3338/AB06dGD58uVMmTKFjIyMSvcvSZKkY88RB+MDUlNTGT16dCR7+UYHvuzXuHFjANatW0dpaSl9+/YN17Rv355WrVqxcuVKzjnnHFauXMmZZ55Js2bNwjUZGRlcf/31bNy4kbPOOouVK1dWOMaBmjFjxhyyj5KSEkpKSsKvi4qKACgtLaW0tDQisx6OhNiD13YfC47kMwrCjBCMOYMwIwRjziDMCMGYMwgzQjDmDMKM1XmumNChvkFXg5SXl/OjH/2IXbt2sXz5cgBycnIYNmxYhZAK0KNHD3r37s0999zDyJEj+fDDDyssiyguLiYxMZEFCxbQr18/TjvtNIYNG8bYsWPDNQsWLCAzM5Pi4mLq1q1b4fgTJkxg4sSJB/WYk5NDvXr1Ijm2JEmSIqC4uJjLL7+cwsJCkpKSvrW20leMq0tWVhYbNmwIh+JoGjt2LNnZ2eHXRUVFtGzZkvT09O/8oCPpjAnH5hroDRMOf3lKEGaEYMwZhBkhGHMGYUYIxpxBmBGCMWcQZjxaB37DfzhqdDAeNWoU8+bNY9myZbRo0SK8PTk5mX379rFr1y4aNWoU3l5QUBB+yl5ycjKrV6+ucLwDd63495r/vJNFQUEBSUlJB10thn89FvtQj7eOi4sjLi6uckNWQklZTLWdK5KO5DMKwowQjDmDMCMEY84gzAjBmDMIM0Iw5gzCjNV5riO6K0V1CYVCjBo1iueff55XX32V1NTUCvu7detGXFwceXl54W2bN29m27ZtpKWlAZCWlsb69evZuXNnuCY3N5ekpCQ6duwYrvn3YxyoOXAMSZIkBUeNvGKclZVFTk4Of/3rX2nQoAE7duwAoGHDhtStW5eGDRtyzTXXkJ2dTePGjUlKSmL06NGkpaVxzjnnAJCenk7Hjh258soruffee9mxYwe33XYbWVlZ4au+1113HVOnTuXmm29m+PDhvPrqqzzzzDPMnz8/arNLkiQpOmrkFePp06dTWFhIr169aN68efjPvz8sZMqUKfTv35+BAwdy4YUXkpyczHPPPRfeHxsby7x584iNjSUtLY0rrriCq666ikmTJoVrUlNTmT9/Prm5uXTu3Jn777+fxx57zFu1SZIkBVCNvGJ8ODfKqFOnDtOmTWPatGnfWNO6dWsWLFjwrcfp1asXb7311hH3KEmSpONLjbxiLEmSJFU3g7EkSZKEwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCDMaSJEkSYDCWJEmSAIOxJEmSBBiMJUmSJMBgLEmSJAEGY0mSJAkwGEuSJEmAwViSJEkCamgwXrZsGZdccgkpKSnExMTwwgsvVNgfCoUYN24czZs3p27duvTt25ctW7ZUqPniiy8YMmQISUlJNGrUiGuuuYY9e/ZUqHn77be54IILqFOnDi1btuTee++t6tEkSZJUQ9XIYLx37146d+7MtGnTDrn/3nvv5fe//z0zZszgjTfeIDExkYyMDL7++utwzZAhQ9i4cSO5ubnMmzePZcuWMXLkyPD+oqIi0tPTad26NevWreO+++5jwoQJzJw5s8rnkyRJUs1TO9oNHEq/fv3o16/fIfeFQiEeeOABbrvtNi699FIA/vCHP9CsWTNeeOEFLrvsMt555x1eeeUV1qxZQ/fu3QF46KGH+OEPf8j//M//kJKSwlNPPcW+fft44okniI+P5/TTTyc/P5/JkydXCNCSJEkKhhoZjL/N1q1b2bFjB3379g1va9iwIT179mTlypVcdtllrFy5kkaNGoVDMUDfvn2pVasWb7zxBj/5yU9YuXIlF154IfHx8eGajIwM7rnnHr788ktOOOGEg85dUlJCSUlJ+HVRUREApaWllJaWVsW4h5QQG6q2c0XSkXxGQZgRgjFnEGaEYMwZhBkhGHMGYUYIxpxBmLE6zxUTCoVq9CcaExPD888/z49//GMAVqxYwXnnncf27dtp3rx5uO7nP/85MTExPP300/zud79jzpw5bN68ucKxmjZtysSJE7n++utJT08nNTWVRx55JLx/06ZNnH766WzatIkOHToc1MuECROYOHHiQdtzcnKoV69ehCaWJElSpBQXF3P55ZdTWFhIUlLSt9Yec1eMo2ns2LFkZ2eHXxcVFdGyZUvS09O/84OOpDMmLKy2c0XShgkZh10bhBkhGHMGYUYIxpxBmBGCMWcQZoRgzBmEGY/Wgd/wH45jLhgnJycDUFBQUOGKcUFBAV26dAnX7Ny5s8L79u/fzxdffBF+f3JyMgUFBRVqDrw+UPOfEhISSEhIOGh7XFwccXFxlRuoEkrKYqrtXJF0JJ9REGaEYMwZhBkhGHMGYUYIxpxBmBGCMWcQZqzOc9XIu1J8m9TUVJKTk8nLywtvKyoq4o033iAtLQ2AtLQ0du3axbp168I1r776KuXl5fTs2TNcs2zZsgrrTnJzc2nXrt0h1xdLkiTp+FYjg/GePXvIz88nPz8f+NcX7vLz89m2bRsxMTGMGTOG3/72t7z44ousX7+eq666ipSUlPA65A4dOvCDH/yAa6+9ltWrV/P6668zatQoLrvsMlJSUgC4/PLLiY+P55prrmHjxo08/fTTPPjggxWWSkiSJCk4auRSirVr19K7d+/w6wNhdejQocyePZubb76ZvXv3MnLkSHbt2sX555/PK6+8Qp06dcLveeqppxg1ahQXXXQRtWrVYuDAgfz+978P72/YsCGLFi0iKyuLbt26ceKJJzJu3Dhv1SZJkhRQNTIY9+rVi2+7WUZMTAyTJk1i0qRJ31jTuHFjcnJyvvU8nTp14m9/+1ul+5QkSdLxo0YupZAkSZKqm8FYkiRJwmAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwlSZIkwGAsSZIkAQZjSZIkCTAYAzBt2jTatGlDnTp16NmzJ6tXr452S5IkSapmgQ/GTz/9NNnZ2YwfP54333yTzp07k5GRwc6dO6PdmiRJkqpR4IPx5MmTufbaaxk2bBgdO3ZkxowZ1KtXjyeeeCLarUmSJKka1Y52A9G0b98+1q1bx9ixY8PbatWqRd++fVm5cuVB9SUlJZSUlIRfFxYWAvDFF19QWlpa9Q3/f7X37622c0XS559/fti1QZgRgjFnEGaEYMwZhBkhGHMGYUYIxpxBmPFo7d69G4BQKPSdtTGhw6k6Tm3fvp2TTz6ZFStWkJaWFt5+8803s3TpUt54440K9RMmTGDixInV3aYkSZKO0kcffUSLFi2+tSbQV4yP1NixY8nOzg6/Li8v54svvqBJkybExMREsbPIKCoqomXLlnz00UckJSVFu50qEYQZIRhzBmFGCMacQZgRgjFnEGaEYMx5PM0YCoXYvXs3KSkp31kb6GB84oknEhsbS0FBQYXtBQUFJCcnH1SfkJBAQkJChW2NGjWqyhajIikp6Zj/h+C7BGFGCMacQZgRgjFnEGaEYMwZhBkhGHMeLzM2bNjwsOoC/eW7+Ph4unXrRl5eXnhbeXk5eXl5FZZWSJIk6fgX6CvGANnZ2QwdOpTu3bvTo0cPHnjgAfbu3cuwYcOi3ZokSZKqUeCD8aBBg/j0008ZN24cO3bsoEuXLrzyyis0a9Ys2q1Vu4SEBMaPH3/QcpHjSRBmhGDMGYQZIRhzBmFGCMacQZgRgjFnEGY8lEDflUKSJEk6INBrjCVJkqQDDMaSJEkSBmNJkiQJMBhLkiRJgMFY/9+0adNo06YNderUoWfPnqxevTraLUXUsmXLuOSSS0hJSSEmJoYXXngh2i1F3F133cXZZ59NgwYNaNq0KT/+8Y/ZvHlztNuKuOnTp9OpU6fwTefT0tJ4+eWXo91Wlbr77ruJiYlhzJgx0W4loiZMmEBMTEyFP+3bt492WxH38ccfc8UVV9CkSRPq1q3LmWeeydq1a6PdVkS1adPmoJ9lTEwMWVlZ0W4tYsrKyrj99ttJTU2lbt26tG3bljvuuIPj8R4Gu3fvZsyYMbRu3Zq6dety7rnnsmbNmmi3VS0MxuLpp58mOzub8ePH8+abb9K5c2cyMjLYuXNntFuLmL1799K5c2emTZsW7VaqzNKlS8nKymLVqlXk5uZSWlpKeno6e/fujXZrEdWiRQvuvvtu1q1bx9q1a+nTpw+XXnopGzdujHZrVWLNmjU88sgjdOrUKdqtVInTTz+dTz75JPxn+fLl0W4por788kvOO+884uLiePnll9m0aRP3338/J5xwQrRbi6g1a9ZU+Dnm5uYC8LOf/SzKnUXOPffcw/Tp05k6dSrvvPMO99xzD/feey8PPfRQtFuLuBEjRpCbm8sf//hH1q9fT3p6On379uXjjz+OdmtVL6TA69GjRygrKyv8uqysLJSSkhK66667othV1QFCzz//fLTbqHI7d+4MAaGlS5dGu5Uqd8IJJ4Qee+yxaLcRcbt37w5973vfC+Xm5oa+//3vh2644YZotxRR48ePD3Xu3DnabVSpW265JXT++edHu41qd8MNN4Tatm0bKi8vj3YrEZOZmRkaPnx4hW0DBgwIDRkyJEodVY3i4uJQbGxsaN68eRW2d+3aNfSb3/wmSl1VH68YB9y+fftYt24dffv2DW+rVasWffv2ZeXKlVHsTEersLAQgMaNG0e5k6pTVlbG3Llz2bt373H5GPesrCwyMzMr/PN5vNmyZQspKSmccsopDBkyhG3btkW7pYh68cUX6d69Oz/72c9o2rQpZ511Fo8++mi026pS+/bt48knn2T48OHExMREu52IOffcc8nLy+Of//wnAH//+99Zvnw5/fr1i3JnkbV//37KysqoU6dOhe1169Y97n6jcyiBf/Jd0H322WeUlZUd9KS/Zs2a8Y9//CNKXelolZeXM2bMGM477zzOOOOMaLcTcevXryctLY2vv/6a+vXr8/zzz9OxY8dotxVRc+fO5c033zyu1/X17NmT2bNn065dOz755BMmTpzIBRdcwIYNG2jQoEG024uI999/n+nTp5Odnc2vf/1r1qxZwy9/+Uvi4+MZOnRotNurEi+88AK7du3i6quvjnYrEXXrrbdSVFRE+/btiY2NpaysjDvvvJMhQ4ZEu7WIatCgAWlpadxxxx106NCBZs2a8ac//YmVK1dy6qmnRru9Kmcwlo5DWVlZbNiw4bj92327du3Iz8+nsLCQP//5zwwdOpSlS5ceN+H4o48+4oYbbiA3N/egqzbHk3+/0tapUyd69uxJ69ateeaZZ7jmmmui2FnklJeX0717d373u98BcNZZZ7FhwwZmzJhx3Abjxx9/nH79+pGSkhLtViLqmWee4amnniInJ4fTTz+d/Px8xowZQ0pKynH3s/zjH//I8OHDOfnkk4mNjaVr164MHjyYdevWRbu1KmcwDrgTTzyR2NhYCgoKKmwvKCggOTk5Sl3paIwaNYp58+axbNkyWrRoEe12qkR8fHz4ykW3bt1Ys2YNDz74II888kiUO4uMdevWsXPnTrp27RreVlZWxrJly5g6dSolJSXExsZGscOq0ahRI0477TTefffdaLcSMc2bNz/oL2wdOnTgL3/5S5Q6qloffvghixcv5rnnnot2KxF30003ceutt3LZZZcBcOaZZ/Lhhx9y1113HXfBuG3btixdupS9e/dSVFRE8+bNGTRoEKecckq0W6tyrjEOuPj4eLp160ZeXl54W3l5OXl5ecflms3jWSgUYtSoUTz//PO8+uqrpKamRrulalNeXk5JSUm024iYiy66iPXr15Ofnx/+0717d4YMGUJ+fv5xGYoB9uzZw3vvvUfz5s2j3UrEnHfeeQfdNvGf//wnrVu3jlJHVWvWrFk0bdqUzMzMaLcSccXFxdSqVTE2xcbGUl5eHqWOql5iYiLNmzfnyy+/ZOHChVx66aXRbqnKecVYZGdnM3ToULp3706PHj144IEH2Lt3L8OGDYt2axGzZ8+eClehtm7dSn5+Po0bN6ZVq1ZR7CxysrKyyMnJ4a9//SsNGjRgx44dADRs2JC6detGubvIGTt2LP369aNVq1bs3r2bnJwclixZwsKFC6PdWsQ0aNDgoLXhiYmJNGnS5LhaM/7f//3fXHLJJbRu3Zrt27czfvx4YmNjGTx4cLRbi5gbb7yRc889l9/97nf8/Oc/Z/Xq1cycOZOZM2dGu7WIKy8vZ9asWQwdOpTatY+/eHHJJZdw55130qpVK04//XTeeustJk+ezPDhw6PdWsQtXLiQUChEu3btePfdd7npppto3779cZULvlG0b4uhmuGhhx4KtWrVKhQfHx/q0aNHaNWqVdFuKaJee+21EHDQn6FDh0a7tYg51HxAaNasWdFuLaKGDx8eat26dSg+Pj500kknhS666KLQokWLot1WlTseb9c2aNCgUPPmzUPx8fGhk08+OTRo0KDQu+++G+22Iu6ll14KnXHGGaGEhIRQ+/btQzNnzox2S1Vi4cKFISC0efPmaLdSJYqKikI33HBDqFWrVqE6deqETjnllNBvfvObUElJSbRbi7inn346dMopp4Ti4+NDycnJoaysrNCuXbui3Va1iAmFjsNHtkiSJElHyDXGkiRJEgZjSZIkCTAYS5IkSYDBWJIkSQIMxpIkSRJgMJYkSZIAg7EkSZIEGIwl6ZiTm5vLsGHDOO2000hKSiIhIYHmzZtz8cUXM2XKFD799NNw7ezZs4mJieHqq6+OXsOSdIw4/p7ZKEnHqc8++4zBgwezePFiANq0aUPv3r1JTExkx44drFixgsWLFzNu3DgWL15Mz549o9yxJB1bDMaSdAwoLCzk/PPPZ/PmzbRv356ZM2dywQUXVKgpKSlhzpw5jB8/nk8++SRKnUrSsctgLEnHgNGjR7N582batGnD66+/TuPGjQ+qSUhIYOTIkVx66aXs2rWr+puUpGOca4wlqYZ7//33ycnJAWDy5MmHDMX/rlmzZrRr1+47j/vcc88xYsQIzjjjDE444QTq1KlDamoqw4cPZ/PmzYd8T0lJCffddx/dunWjQYMGxMfHk5yczNlnn83NN9/MF198UaF+y5YtDB8+nNTUVBISEqhfvz6tW7cmMzOTWbNmHeYnIEnVwyvGklTDzZs3j7KyMho1asSPfvSjiB335z//OQkJCXTs2JE+ffqwf/9+NmzYwKxZs3jmmWdYtGgR5557bri+vLyczMxM8vLySEpK4oILLqBRo0Z8+umnbNmyhfvuu4/LL788HNw3bNjAeeedR1FREe3ataN///7Exsbyv//7vyxbtoyPP/6YYcOGRWweSTpaBmNJquHWrl0LQNeuXYmNjY3YcZ966in69+9PYmJieFsoFGL69OlkZWUxcuRI1q9fT0xMDADLly8nLy+Ps846i6VLl9KgQYOD+mzZsmX49eTJkykqKuK3v/0tv/nNbyrUfvXVV6xZsyZis0hSJLiUQpJquAO3X2vatGlEjzto0KAKoRggJiaGX/ziF6SlpbFx40beeeed8L6CggIALrjggoNCMUD37t1p0qTJQfU//OEPD6qtW7cuF154YUTmkKRI8YqxJAXYu+++yyuvvMK7777L7t27KSsrA/4v1G7evJmOHTsC/3fF+oknnuC0005jwIABNG/e/BuP3aNHDxYsWMD111/PxIkT+f73v0+dOnWqfihJqiSDsSTVcCeddBIAO3fujNgxy8rKGDVqFI888gihUOgb64qKisL/v23btkyZMoWbbrqJUaNGMWrUKFq3bk1aWhr9+/fnZz/7GfHx8eH6m266ieXLl7N48WJ+8IMfEBcXR+fOnbnwwgu57LLLOPvssyM2jyRFgkspJKmG69atGwBvvvlm+Iru0XrwwQeZMWMGzZo1Iycnhw8++ICvvvqKUChEKBRi8ODBAAeF5tGjR/Phhx8yc+ZMrrrqKmJjY5k7dy5XXHEFHTt2rHD/5Hr16pGbm8vq1auZNGkSF110Ef/85z+ZPHkyPXr0ICsrKyKzSFKkGIwlqYbr378/tWrVYteuXbz44osROeYzzzwDwCOPPMLgwYNp3bp1hWUOW7Zs+cb3NmvWjGuvvZY5c+bw3nvv8c4775CWlsZ7773HrbfeelD92Wefze23387LL7/M559/zrPPPkvdunV5+OGHee211yIyjyRFgsFYkmq4tm3bhq/g/upXvzroXsH/aefOnd94H+IDDhyjdevWB+3buHEj+fn5h91f+/btueWWWwC+8321a9fmpz/9KRkZGYdVL0nVyWAsSceAhx56iFNPPZWtW7dy/vnns3z58oNq9u3bxxNPPMFZZ51V4W4Sh9KhQwcApk2bRnl5eXj7J598wlVXXcX+/fsPes+rr77KggULKC0trbA9FAoxb948oGLQfvjhhw8Z0Hfs2BG+Bd2hgrkkRYtfvpOkY8AJJ5zA66+/zqBBg1iyZAkXXHABqampdOrUiXr16lFQUMDq1avZs2cPSUlJpKSkfOvxfv3rX/PKK6/w6KOP8tprr9G1a1eKiopYunQpp5xyCj/5yU94/vnnK7zn7bff5sYbbyQpKYmuXbuSkpLCV199xZtvvsmHH35Iw4YNmTRpUrh+5syZZGVlkZqayhlnnEFSUhKffvopf/vb3/jqq6/o06dPRB9YIklHy2AsSceIpk2b8tprr/HKK6/wpz/9iRUrVpCXl0dJSQlNmjQhLS2NzMxMrrzyyu98bHTPnj1Zu3Ytt912G2vWrOHFF1+kZcuWjB49mttuu43Ro0cf9J5LLrmEwsJC/va3v7FlyxZWrVpF3bp1admyJbfeeitZWVm0aNEiXH/nnXcyf/58Vq1axapVqygsLKRp06b07NmTYcOGMXjwYGrX9j9DkmqOmNC33adHkiRJCgjXGEuSJEkYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJgMFYkiRJAgzGkiRJEmAwliRJkgCDsSRJkgQYjCVJkiTAYCxJkiQBBmNJkiQJMBhLkiRJAPw/hxpsJwyEqLcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = (8, 5)\n",
    "plt.bar(train_df['label'].value_counts().index, train_df['label'].value_counts())\n",
    "plt.xticks(np.arange(n_class))\n",
    "plt.xlabel('Class', fontsize=16)\n",
    "plt.ylabel('Count', fontsize=16)\n",
    "plt.grid('on', axis='y')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNIST_data(Dataset):\n",
    "    \"\"\"MNIST dtaa set\"\"\"\n",
    "    \n",
    "    def __init__(self, file_path, \n",
    "                 transform = transforms.Compose([transforms.ToPILImage(), transforms.ToTensor(), \n",
    "                     transforms.Normalize(mean=(0.5,), std=(0.5,))])\n",
    "                ):\n",
    "        \n",
    "        df = pd.read_csv(file_path)\n",
    "        \n",
    "        if len(df.columns) == n_pixels:\n",
    "            # test data\n",
    "            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n",
    "            self.y = None\n",
    "        else:\n",
    "            # training data\n",
    "            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n",
    "            self.y = torch.from_numpy(df.iloc[:,0].values)\n",
    "            \n",
    "        self.transform = transform\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if self.y is not None:\n",
    "            return self.transform(self.X[idx]), self.y[idx]\n",
    "        else:\n",
    "            return self.transform(self.X[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomRotation(object):\n",
    "    \"\"\"\n",
    "    https://github.com/pytorch/vision/tree/master/torchvision/transforms\n",
    "    Rotate the image by angle.\n",
    "    Args:\n",
    "        degrees (sequence or float or int): Range of degrees to select from.\n",
    "            If degrees is a number instead of sequence like (min, max), the range of degrees\n",
    "            will be (-degrees, +degrees).\n",
    "        resample ({PIL.Image.NEAREST, PIL.Image.BILINEAR, PIL.Image.BICUBIC}, optional):\n",
    "            An optional resampling filter.\n",
    "            See http://pillow.readthedocs.io/en/3.4.x/handbook/concepts.html#filters\n",
    "            If omitted, or if the image has mode \"1\" or \"P\", it is set to PIL.Image.NEAREST.\n",
    "        expand (bool, optional): Optional expansion flag.\n",
    "            If true, expands the output to make it large enough to hold the entire rotated image.\n",
    "            If false or omitted, make the output image the same size as the input image.\n",
    "            Note that the expand flag assumes rotation around the center and no translation.\n",
    "        center (2-tuple, optional): Optional center of rotation.\n",
    "            Origin is the upper left corner.\n",
    "            Default is the center of the image.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, degrees, resample=False, expand=False, center=None):\n",
    "        if isinstance(degrees, numbers.Number):\n",
    "            if degrees < 0:\n",
    "                raise ValueError(\"If degrees is a single number, it must be positive.\")\n",
    "            self.degrees = (-degrees, degrees)\n",
    "        else:\n",
    "            if len(degrees) != 2:\n",
    "                raise ValueError(\"If degrees is a sequence, it must be of len 2.\")\n",
    "            self.degrees = degrees\n",
    "\n",
    "        self.resample = resample\n",
    "        self.expand = expand\n",
    "        self.center = center\n",
    "\n",
    "    @staticmethod\n",
    "    def get_params(degrees):\n",
    "        \"\"\"Get parameters for ``rotate`` for a random rotation.\n",
    "        Returns:\n",
    "            sequence: params to be passed to ``rotate`` for random rotation.\n",
    "        \"\"\"\n",
    "        angle = np.random.uniform(degrees[0], degrees[1])\n",
    "\n",
    "        return angle\n",
    "\n",
    "    def __call__(self, img):\n",
    "        \"\"\"\n",
    "            img (PIL Image): Image to be rotated.\n",
    "        Returns:\n",
    "            PIL Image: Rotated image.\n",
    "        \"\"\"\n",
    "        \n",
    "        def rotate(img, angle, resample=False, expand=False, center=None):\n",
    "            \"\"\"Rotate the image by angle and then (optionally) translate it by (n_columns, n_rows)\n",
    "            Args:\n",
    "            img (PIL Image): PIL Image to be rotated.\n",
    "            angle ({float, int}): In degrees degrees counter clockwise order.\n",
    "            resample ({PIL.Image.NEAREST, PIL.Image.BILINEAR, PIL.Image.BICUBIC}, optional):\n",
    "            An optional resampling filter.\n",
    "            See http://pillow.readthedocs.io/en/3.4.x/handbook/concepts.html#filters\n",
    "            If omitted, or if the image has mode \"1\" or \"P\", it is set to PIL.Image.NEAREST.\n",
    "            expand (bool, optional): Optional expansion flag.\n",
    "            If true, expands the output image to make it large enough to hold the entire rotated image.\n",
    "            If false or omitted, make the output image the same size as the input image.\n",
    "            Note that the expand flag assumes rotation around the center and no translation.\n",
    "            center (2-tuple, optional): Optional center of rotation.\n",
    "            Origin is the upper left corner.\n",
    "            Default is the center of the image.\n",
    "            \"\"\"\n",
    "                \n",
    "            return img.rotate(angle, resample, expand, center)\n",
    "\n",
    "        angle = self.get_params(self.degrees)\n",
    "\n",
    "        return rotate(img, angle, self.resample, self.expand, self.center)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomShift(object):\n",
    "    def __init__(self, shift):\n",
    "        self.shift = shift\n",
    "        \n",
    "    @staticmethod\n",
    "    def get_params(shift):\n",
    "        \"\"\"Get parameters for ``rotate`` for a random rotation.\n",
    "        Returns:\n",
    "            sequence: params to be passed to ``rotate`` for random rotation.\n",
    "        \"\"\"\n",
    "        hshift, vshift = np.random.uniform(-shift, shift, size=2)\n",
    "\n",
    "        return hshift, vshift \n",
    "    def __call__(self, img):\n",
    "        hshift, vshift = self.get_params(self.shift)\n",
    "        \n",
    "        return img.transform(img.size, Image.AFFINE, (1,0,hshift,0,1,vshift), resample=Image.BICUBIC, fill=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the Data into Tensors\n",
    "For the training set, apply random rotation within the range of (-45, 45) degrees, shift by (-3, 3) pixels and normalize pixel values to [-1, 1]. For the test set, only apply nomalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "\n",
    "train_dataset = MNIST_data('../datasets/digit-recognizer/train.csv', transform= transforms.Compose(\n",
    "                            [transforms.ToPILImage(), RandomRotation(degrees=20), RandomShift(3),\n",
    "                             transforms.ToTensor(), transforms.Normalize(mean=(0.5,), std=(0.5,))]))\n",
    "test_dataset = MNIST_data('../datasets/digit-recognizer/test.csv')\n",
    "\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                           batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:17: DeprecationWarning: AFFINE is deprecated and will be removed in Pillow 10 (2023-07-01). Use Transform.AFFINE instead.\n",
      "  app.launch_new_instance()\n",
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:17: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwoAAAEWCAYAAADGs2DoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0DUlEQVR4nO3deXQUZdr38V8nkA5bEtkSIiGy74ITAREUhMjOgMIo+jwKjuiIgVdA0EEdVjWKjiuI67ANqKACCsoiqwyLI8rjIAqCgCiGTZJAkATI/f7hSY9tpavTnQ698P2cU+eYuuquurqwr86Vqr7LYYwxAgAAAIDfiAp2AgAAAABCD40CAAAAAAsaBQAAAAAWNAoAAAAALGgUAAAAAFjQKAAAAACwoFEAAAAAYEGjAAAAAMCCRgEAAACABY1CBBgyZIguu+yyYKcRFiZOnCiHwxHsNICQFQ71ZN26dXI4HHrnnXe8blvc6zl16pSGDh2qpKQkORwOjRw5smwSBYAwR6Pgo1mzZsnhcLiWcuXK6dJLL9WQIUP0448/Bju9C27IkCFu58PpdKpRo0YaP368zpw549c+58+fr+eee87vnE6fPq2JEydq3bp1fu8DuBCoJ1YffPCBOnXqpJo1a6pixYqqV6+ebrrpJi1fvjxgx3j88cc1a9YsDRs2THPnztVtt92mTZs2aeLEicrOzg7YcYALae/evfrLX/6ievXqKTY2VnFxcerQoYOef/55/fLLL8FOD2GqXLATCFeTJ09W3bp1debMGW3ZskWzZs3Sxo0btWPHDsXGxgY7vQvK6XTq9ddflyTl5ORoyZIlmjJlivbu3at58+b5vL/58+drx44dfv+V7/Tp05o0aZIkqXPnzm6xRx55RH/961/92i9QVqgnv3r66ac1duxYderUSePGjVPFihW1Z88effzxx3rrrbfUo0cPn/f52muvqbCw0G3dmjVrdNVVV2nChAlux540aZKGDBmihISE0r4U4IJatmyZ/vSnP8npdOr2229XixYtVFBQoI0bN2rs2LH66quv9OqrrwY7TYQhGgU/9ezZU1deeaUkaejQoapevbqefPJJvf/++7rpppuCnN2FVa5cOf3v//6v6+d7771XV199td58800988wzSkxMDGJ27sqVK6dy5fjfHqGFeiKdO3dOU6ZM0fXXX6+VK1da4keOHPFrv+XLly92X82aNfNrf0Co2bdvnwYNGqTU1FStWbNGtWrVcsUyMjK0Z88eLVu2LIgZIpxx61GAXHPNNZJ+vfQnSQUFBRo/frzS0tIUHx+vSpUq6ZprrtHatWvdxu3fv18Oh0NPP/20Xn31VdWvX19Op1Nt2rTRv//9b8txFi9erBYtWig2NlYtWrTQokWLis0nLy9P999/v1JSUuR0OtW4cWM9/fTTMsa4bedwODR8+HAtXLhQzZo1U4UKFdS+fXv95z//kSS98soratCggWJjY9W5c2ft37/f67lwOBzq2LGjjDH67rvv3GIvvfSSmjdvLqfTqeTkZGVkZLhd6u/cubOWLVumAwcOuG7HKLq/uCTndP/+/apRo4YkadKkSa59TJw4UVLx31Eo+gWl6Nxfdtlleuihh5Sfn++23WWXXaY+ffpo48aNatu2rWJjY1WvXj3NmTPH6zkBfHEx1pNjx44pNzdXHTp0KDaHmjVrWtYVFhbqscceU+3atRUbG6uuXbtqz549btv89jsKRd9t2Ldvn5YtW+aqD0OGDNHYsWMlSXXr1nWtL0m9A4Jt6tSpOnXqlN544w23JqFIgwYNdN9990ny/fNu3bp1uvLKK1WhQgW1bNnSdUvve++9p5YtWyo2NlZpaWn64osv3MYPGTJElStX1nfffafu3burUqVKSk5O1uTJky11o6T1ZdWqVerYsaMSEhJUuXJlNW7cWA899JDbNvn5+ZowYYIaNGggp9OplJQUPfDAA5bXBx8Y+GTmzJlGkvn3v//ttn7atGlGkpkxY4YxxpijR4+aWrVqmdGjR5sZM2aYqVOnmsaNG5vy5cubL774wjVu3759RpK54oorTIMGDcyTTz5ppk6daqpXr25q165tCgoKXNuuWLHCREVFmRYtWphnnnnGPPzwwyY+Pt40b97cpKamurYrLCw0Xbp0MQ6HwwwdOtRMmzbN9O3b10gyI0eOdMtbkrn88stNSkqKeeKJJ8wTTzxh4uPjTZ06dcy0adNMs2bNzN///nfzyCOPmJiYGHPddde5jR88eLCpVKmS5TwNHDjQSDJff/21a92ECROMJJOenm5efPFFM3z4cBMdHW3atGnjep0rV640rVu3NtWrVzdz5841c+fONYsWLSrxOT116pSZMWOGkWRuuOEG1z7+7//+zy2H378GSWbgwIFm+vTp5vbbbzeSTP/+/d22S01NNY0bNzaJiYnmoYceMtOmTTN/+MMfjMPhMDt27LCcA8Ab6sl/68n58+dNhQoVTFpamjl+/LjteVu7dq3rdaalpZlnn33WTJw40VSsWNG0bdvWbdvBgwe7Xk9WVpaZO3euqV69umndurWrPmzfvt3ccsstRpJ59tlnXetPnTrl9d8QCLZLL73U1KtXr0Tb+vp5V6tWLTNx4kTz7LPPmksvvdRUrlzZ/POf/zR16tRxe483aNDAnD9/3u04sbGxpmHDhua2224z06ZNM3369DGSzN/+9jfXdiWtLzt27DAxMTHmyiuvNM8//7x5+eWXzZgxY8y1117r2ub8+fOmW7dupmLFimbkyJHmlVdeMcOHDzflypUz/fr18/PsgkbBR0Uf7B9//LE5evSoOXjwoHnnnXdMjRo1jNPpNAcPHjTGGHPu3DmTn5/vNvbEiRMmMTHR/PnPf3atK/pgr1atmvn5559d65csWWIkmQ8++MC1rnXr1qZWrVomOzvbtW7lypVGktsH++LFi40k8+ijj7odf+DAgcbhcJg9e/a41kkyTqfT7Nu3z7XulVdeMZJMUlKSyc3Nda0fN26ckeS2bVGjcPToUXP06FGzZ88e8/TTTxuHw2FatGhhCgsLjTHGHDlyxMTExJhu3bq5FZOiX4j+8Y9/uNb17t3b7fUUKek5PXr0qJFkJkyYYNnH7xuF7du3G0lm6NChbtuNGTPGSDJr1qxxrUtNTTWSzIYNG1zrjhw5YpxOp7n//vstxwK8oZ6415Px48cbSaZSpUqmZ8+e5rHHHjPbtm2znLeiRqFp06Zu5+X55583ksx//vMf17rfNgpFUlNTTe/evd3WPfXUU5Z8gFCXk5NjJJXoF2F/Pu82bdrkWrdixQojyVSoUMEcOHDAtb7oPb527VrXuqKGZMSIEa51hYWFpnfv3iYmJsYcPXrUGFPy+vLss88aSa5xxZk7d66Jiooyn3zyidv6l19+2Ugy//rXv7ydIhSDW4/8lJ6erho1aiglJUUDBw5UpUqV9P7776t27dqSpOjoaMXExEj69fL4zz//rHPnzunKK6/U559/btnfzTffrEsuucT1c9GtB0W37vz000/avn27Bg8erPj4eNd2119/veVe2w8//FDR0dH6f//v/7mtv//++2WM0UcffeS2vmvXrm7TB7Zr106SNGDAAFWpUsWy/ve3E+Xl5alGjRqqUaOGGjRooDFjxqhDhw5asmSJ6zafjz/+WAUFBRo5cqSiov77v91dd92luLi4Et0/6es5LYkPP/xQkjR69Gi39ffff78kWfJq1qyZ699GkmrUqKHGjRtbzgngC+rJryZNmqT58+friiuu0IoVK/Twww8rLS1Nf/jDH/T1119bXucdd9zhOi/FvU4g0uXm5kqS23vLE38+79q3b+/6ueg926VLF9WpU8eyvrj33fDhw13/XXRrYkFBgT7++GNXTiWpL0UTDCxZssQyOUGRhQsXqmnTpmrSpImOHTvmWrp06SJJlls1UTI0Cn6aPn26Vq1apXfeeUe9evXSsWPH5HQ63baZPXu2Lr/8csXGxqpatWqqUaOGli1bppycHMv+fvumk+T6kD9x4oQk6cCBA5Kkhg0bWsY2btzY7ecDBw4oOTnZUjiaNm3qti9Pxy76xSElJaXY9UU5FYmNjdWqVau0atUqzZw5U02bNtWRI0dUoUIFt5yKyzUmJkb16tWz5OSJL+e0JA4cOKCoqCg1aNDAbX1SUpISEhK8nivp13+r358TwBfUk/+65ZZb9Mknn+jEiRNauXKlbr31Vn3xxRfq27evZcplb68TiHRxcXGSpJMnT3rdtrSfd76+l6OiolSvXj23dY0aNZIk1/d/Slpfbr75ZnXo0EFDhw5VYmKiBg0apAULFrg1Dd9++62++uor1x8ui5aiY/o7IcLFjulf/NS2bVvXLCX9+/dXx44ddeutt2rXrl2qXLmy/vnPf2rIkCHq37+/xo4dq5o1ayo6OlqZmZmuLyj+VnR0dLHHMb/7Mk9Z8HTskuYUHR2t9PR018/du3dXkyZN9Je//EXvv/9+wPL09Zz6oqQPYQvmvxMiF/XEKi4uTtdff72uv/56lS9fXrNnz9bWrVvVqVMnv/cJRJq4uDglJydrx44dJR5T2s+7YLzvKlSooA0bNmjt2rVatmyZli9frrfffltdunTRypUrFR0drcLCQrVs2VLPPPNMsfv4fYODkuGKQgAUfWAfOnRI06ZNkyS98847qlevnt577z3ddttt6t69u9LT0/1+CFlqaqqkXzvm39u1a5dl20OHDln+wvDNN9+47aus1KpVS6NGjdIHH3ygLVu2uB3z97kWFBRo3759bjl5KmIlPae+PHk5NTVVhYWFlvN6+PBhZWdnl/m5An6PemJV1ET99NNPZXYMntiOcNWnTx/t3btXmzdvtt3uQn/eFRYWWm5H2r17tyS5bk/0pb5ERUWpa9eueuaZZ7Rz50499thjWrNmjeuWovr16+vnn39W165dlZ6ebll+f7UUJUOjECCdO3dW27Zt9dxzz+nMmTOujvu3HfbWrVu9vpE9qVWrllq3bq3Zs2e73WqwatUq7dy5023bXr166fz5865fMoo8++yzcjgc6tmzp185+GLEiBGqWLGinnjiCUm/3oMdExOjF154we2cvPHGG8rJyVHv3r1d6ypVqlTs7RQlPacVK1aUpBI9YbVXr16SZHkSdNFfJH6bF3ChXIz15PTp0x5fT9F9ymX5QV+pUiVJJasbQCh54IEHVKlSJQ0dOlSHDx+2xPfu3avnn38+KJ93v60bxhhNmzZN5cuXV9euXSWVvL78/PPPln23bt1aklxTn95000368ccf9dprr1m2/eWXX5SXlxeQ13Sx4dajABo7dqz+9Kc/adasWerTp4/ee+893XDDDerdu7f27dunl19+Wc2aNdOpU6f82n9mZqZ69+6tjh076s9//rN+/vlnvfjii2revLnbPvv27avrrrtODz/8sPbv369WrVpp5cqVWrJkiUaOHKn69esH6iV7VK1aNd1xxx166aWX9PXXX6tp06YaN26cJk2apB49euiPf/yjdu3apZdeeklt2rRxe2BbWlqa3n77bY0ePVpt2rRR5cqV1bdv3xKf0woVKqhZs2Z6++231ahRI1WtWlUtWrRQixYtLHm2atVKgwcP1quvvqrs7Gx16tRJn376qWbPnq3+/fvruuuuK/NzBRTnYqsnp0+f1tVXX62rrrpKPXr0UEpKirKzs7V48WJ98skn6t+/v6644oqAHKs4aWlpkqSHH35YgwYNUvny5dW3b19XAwGEqvr162v+/Pm6+eab1bRpU7cnM2/atEkLFy7UkCFDdN99913Qz7vY2FgtX75cgwcPVrt27fTRRx9p2bJleuihh1zPOyppfZk8ebI2bNig3r17KzU1VUeOHNFLL72k2rVrq2PHjpKk2267TQsWLNA999yjtWvXqkOHDjp//ry++eYbLViwQCtWrHBdnYQPgjDTUljzNO+5Mb/O4Vu/fn1Tv359c+7cOfP444+b1NRU43Q6zRVXXGGWLl1qmaqvaDrDp556yrI/FTPF57vvvmuaNm1qnE6nadasmXnvvfeKnf7v5MmTZtSoUSY5OdmUL1/eNGzY0Dz11FOu6Up/e4yMjAy3dZ5yKpqScOHCha51np6jYIwxe/fuNdHR0Wbw4MGuddOmTTNNmjQx5cuXN4mJiWbYsGHmxIkTbuNOnTplbr31VpOQkOA2VWNhYWGJzqkxxmzatMmkpaWZmJgYt/NY3HMUzp49ayZNmmTq1q1rypcvb1JSUsy4cePMmTNn3LYrbkpFY4zp1KmT6dSpU7HnALBDPflvPTl79qx57bXXTP/+/V2vs2LFiuaKK64wTz31lNs0qMXVot8ea+bMma51JZ0e1RhjpkyZYi699FITFRXFVKkIO7t37zZ33XWXueyyy0xMTIypUqWK6dChg3nxxRddn2el/bwr6Xu86HeDvXv3up5tkJiYaCZMmOA2RboxJasvq1evNv369TPJyckmJibGJCcnm1tuucXs3r3bbV8FBQXmySefNM2bNzdOp9NccsklJi0tzUyaNMnk5OT4d2Ivcg5j+NYXAAAAAmPIkCF65513/L7iidDBdxQAAAAAWNAoAAAAALCgUQAAAABgwXcUAAAAAFhwRQEAAACABY0CAAAAAIuQe+BaYWGhDh06pCpVqsjhcAQ7HSDsGWN08uRJJScnKyoqvP42QD0AAitc6wG1AAisEteCsnpAw7Rp01wPzWnbtq3ZunVricYdPHjQSGJhYQnwcvDgwbJ6u9vytxYYQz1gYSmrJdzqAbWAhaVsFm+1oEwahbfeesvExMSYf/zjH+arr74yd911l0lISDCHDx/2OjY7OzvoJ42FJRKX7Ozssni72ypNLTCGesDCUlZLuNUDagELS9ks3mpBmTQKbdu2dXvE9/nz501ycrLJzMy0bHvmzBmTk5PjWvirAQtL2SzBeHy9L7XAGOoBC8uFWkK9HlALWFguzOKtFgT8BsWCggJt27ZN6enprnVRUVFKT0/X5s2bLdtnZmYqPj7etaSkpAQ6JQBB4GstkKgHQKTidwMgPAW8UTh27JjOnz+vxMREt/WJiYnKysqybD9u3Djl5OS4loMHDwY6JQBB4GstkKgHQKTidwMgPAV91iOn0ymn0xnsNACEAOoBAIlaAISKgF9RqF69uqKjo3X48GG39YcPH1ZSUlKgDwcgRFELABShHgDhKeCNQkxMjNLS0rR69WrXusLCQq1evVrt27cP9OEAhChqAYAi1AMgTAV8WgPz6xRoTqfTzJo1y+zcudPcfffdJiEhwWRlZXkdm5OTE/RvgLOwROISjFlOSlMLjKEesLCU1RJu9YBawMJSNou3WlAm31G4+eabdfToUY0fP15ZWVlq3bq1li9fbvkSE4DIRi0AUIR6AIQfhzHGBDuJ38rNzVV8fHyw0wAiTk5OjuLi4oKdhk+oB0DZCLd6QC0Ayoa3WhDw7ygAAAAACH80CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABgQaMAAAAAwIJGAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABgQaMAAAAAwIJGAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsygU7AZSeMcY2XlhY6Pe+P//8c4+xSZMm2Y5dunSp38cFUDZatGjhMTZu3DjbsVWrVvUYi4qy/7vTDz/8YBufMWOGx9hnn31mOxYAUDa4ogAAAADAgkYBAAAAgAWNAgAAAAALGgUAAAAAFjQKAAAAACxoFAAAAABY0CgAAAAAsOA5CiEiMTHRY2zYsGG2Y709J8HbcxbsXHHFFR5jr732mu1Yu/naJen48eN+5QSEO4fDYRu3e8/Gxsbajn3sscds47179/YYa9y4se3YstSpUyePsS5dutiO/f777wOdDhAS6tev7zH217/+1XZsUlKS38c9cuSIbdzu83/Lli1+HxehJ+BXFCZOnCiHw+G2NGnSJNCHARDiqAUAilAPgPBUJlcUmjdvro8//vi/BynHhQvgYkQtAFCEegCEnzJ5l5YrV67El7zy8/OVn5/v+jk3N7csUgIQBL7UAol6AEQyfjcAwk+ZfJn522+/VXJysurVq6f/+Z//sb1/NDMzU/Hx8a4lJSWlLFICEAS+1AKJegBEMn43AMJPwBuFdu3aadasWVq+fLlmzJihffv26ZprrtHJkyeL3X7cuHHKyclxLQcPHgx0SgCCwNdaIFEPgEjF7wZAeAr4rUc9e/Z0/ffll1+udu3aKTU1VQsWLNCdd95p2d7pdMrpdAY6DQBB5mstkKgHQKTidwMgPJX5cxQSEhLUqFEj7dmzp6wPBSCEUQsAFKEeAOGhzKccOHXqlPbu3avbbrutrA8VdKX568fVV1/tMXbvvffajvX2nIRZs2Z5jL3++uu2Yxs1auQxNn36dNux3uZhnjZtmsfYfffdZzsW4ediqgXeeHvP2tWDRx991Hbsdddd51dOkvfnEcyePdtjzNuzIcaMGWMbt5sv3tMVqCJTp061jefl5dnGceGFUz0ozXNPJCkqyvPfZCdMmGA79sYbb/QY8/asorLUtWtXj7Fu3brZjt29e3eg00EZCvgVhTFjxmj9+vXav3+/Nm3apBtuuEHR0dG65ZZbAn0oACGMWgCgCPUACE8Bv6Lwww8/6JZbbtHx48dVo0YNdezYUVu2bFGNGjUCfSgAIYxaAKAI9QAITwFvFN56661A7xJAGKIWAChCPQDCU5l/mRkAAABA+KFRAAAAAGBBowAAAADAosynR72Y5Ofne4yNHTvWdmxmZqbfx/U21djQoUP93veWLVs8xlJTU23HTpw40TbOlIVA8WrVquUx1qRJE9uxdtMOS9KaNWs8xr777jvbsbt27fIYO3PmjO3Y3Nxc27hdDbzqqqtsx1aqVMk2Tq1BaXib/rR169a28SlTpniM9enTx5+UJElZWVm28Tlz5tjG7X5nGTVqlO1Yu8//u+66y3bs5MmTbeOentaN4OCKAgAAAAALGgUAAAAAFjQKAAAAACxoFAAAAABY0CgAAAAAsKBRAAAAAGBBowAAAADAgucoXCCHDh2yjS9btsxjzG4OZkn69ttv/cqptBo1amQbdzgctvHPP/88kOkAEcPuWQe9evWyHbtnzx7b+KlTp/zKqbROnDhhGy8sLLxAmQCBdemll9rG09LSPMZmz55tO/bDDz/0GPP23JOvvvrKNv7LL794jB07dsx27N///nePMW/PPUlISLCN8xyF0MIVBQAAAAAWNAoAAAAALGgUAAAAAFjQKAAAAACwoFEAAAAAYEGjAAAAAMCC6VEvkHnz5pUqHizVqlXzGEtNTbUda4yxja9du9avnIBIZzeVqLdpRkOVt+mUy5cv7zG2e/du27H5+fl+5QQEwqZNm2zjffv29Rjbu3ev7djs7Gx/Uiq1I0eO2MbPnTt3gTJBsHFFAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABgwXMUYKtLly4eY1dffbXtWG/zMB8/ftyvnACEnksuucQ23rlzZ7/3vWbNGtt4Tk6O3/sGSsvbs022bdt2gTIJHG/PPYmNjfUY++6772zHnj592q+cEBw+X1HYsGGD+vbtq+TkZDkcDi1evNgtbozR+PHjVatWLVWoUEHp6en69ttvA5UvgBBBLQAgUQuASOZzo5CXl6dWrVpp+vTpxcanTp2qF154QS+//LK2bt2qSpUqqXv37jpz5kypkwUQOqgFACRqARDJfL71qGfPnurZs2exMWOMnnvuOT3yyCPq16+fJGnOnDlKTEzU4sWLNWjQIMuY/Px85efnu37Ozc31NSUAQRDoWiBRD4BwRC0AIldAv8y8b98+ZWVlKT093bUuPj5e7dq10+bNm4sdk5mZqfj4eNeSkpISyJQABIE/tUCiHgCRhloAhLeANgpZWVmSpMTERLf1iYmJrtjvjRs3Tjk5Oa7l4MGDgUwJQBD4Uwsk6gEQaagFQHgL+qxHTqdTTqcz2GkACAHUAwAStQAIFQG9opCUlCRJOnz4sNv6w4cPu2IAIh+1AIBELQDCXUCvKNStW1dJSUlavXq1WrduLenXLyBt3bpVw4YNC+ShcIH861//8hjbuXOn7dhmzZoFOh2ECWrBxWfUqFG28TZt2tjGFyxY4DG2ceNGv3JC8FELQleFChU8xkrz3JO1a9faxnmGUnjxuVE4deqU9uzZ4/p537592r59u6pWrao6depo5MiRevTRR9WwYUPVrVtXf/vb35ScnKz+/fsHMm8AQUYtACBRC4BI5nOj8Nlnn+m6665z/Tx69GhJ0uDBgzVr1iw98MADysvL0913363s7Gx17NhRy5cvt32KH4DwQy0AIFELgEjmc6PQuXNnGWM8xh0OhyZPnqzJkyeXKjEAoY1aAECiFgCRLKBfZgYAAAAQGWgUAAAAAFjQKAAAAACwCPoD1xBcCQkJtvFXXnnFY6x58+a2Yw8dOuRPSgBC1MCBAz3GSjvV5QsvvOAxdvTo0VLtG4BVRkaGx9hvv5xenBUrVniMrVq1yu+cEHq4ogAAAADAgkYBAAAAgAWNAgAAAAALGgUAAAAAFjQKAAAAACxoFAAAAABY0CgAAAAAsOA5CheB5ORkj7E1a9bYjm3QoIHH2KZNm2zHduvWzT4xAD5zOBy2cWOM3/seNWqUbfzBBx/0GIuOjrYd6+05C1u3brWNA/CNt89gb+93O88//7zH2I8//uj3fhF6uKIAAAAAwIJGAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWDA96kVg48aNHmOpqam2Y0sz1aK3aRztNGrUyDa+e/duv/cNhLqoKM9/wyksLLQdazcdsiQNHz7cY+zee++1HRsfH+8xtn//ftuxr7zyim28NLUGCGWleT/bGTp0qG18woQJtvG4uDiPsQceeMB2rLep1RE5uKIAAAAAwIJGAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWNAoAAAAALDgOQoR4J577rGN16lTp0yO2759e9v4rl27bOMnT570GGvcuLHt2Iceesg2/uSTT9rGgVBmN7d67969bceOHTvWNt6pUye/cvKmdu3atvFFixbZxvPy8jzGRo8ebTv28OHDtnGgLEVHR9vGz58/7zFWtWpV27HDhg3zGBs1apTt2GrVqtnGs7KyPMamT59uOzY/P982jsjh8xWFDRs2qG/fvkpOTpbD4dDixYvd4kOGDJHD4XBbevToEah8AYQIagEAiVoARDKfG4W8vDy1atXKttvs0aOHfvrpJ9fy5ptvlipJAKGHWgBAohYAkcznW4969uypnj172m7jdDqVlJRUov3l5+e7XcLKzc31NSUAQRDoWiBRD4BwRC0AIleZfJl53bp1qlmzpho3bqxhw4bp+PHjHrfNzMxUfHy8a0lJSSmLlAAEgS+1QKIeAJGKWgCEp4A3Cj169NCcOXO0evVqPfnkk1q/fr169uzp8cs848aNU05Ojms5ePBgoFMCEAS+1gKJegBEImoBEL4CPuvRoEGDXP/dsmVLXX755apfv77WrVunrl27WrZ3Op1yOp2BTgNAkPlaCyTqARCJqAVA+Crz5yjUq1dP1atX1549e8r6UABCGLUAgEQtAMJJmT9H4YcfftDx48dVq1atsj5UxJoxY4ZtfPDgwX7ve8yYMbZxuznbW7VqZTs2OTnZr5wk6fPPP7eNP/bYY7Zxu7mn09LSbMd6u3cW/qEWBEbnzp1t496ek2D3pdCVK1fajm3ZsqXHmLdnn/Tr1882bqddu3a2cW/1YN68eR5jBQUFfuUE/0VaLbC7hUqSrrvuOo+xBx54wHZsWU4jW7NmTY+x+fPn2461qyOPPPKI7djvv//ePjGEFJ8bhVOnTrn9FWDfvn3avn27qlatqqpVq2rSpEkaMGCAkpKStHfvXj3wwANq0KCBunfvHtDEAQQXtQCARC0AIpnPjcJnn33m1h0XPTFz8ODBmjFjhr788kvNnj1b2dnZSk5OVrdu3TRlyhTuNQQiDLUAgEQtACKZz41C586dZYzxGF+xYkWpEgIQHqgFACRqARDJyvzLzAAAAADCD40CAAAAAAsaBQAAAAAWDmN3Y2EQ5ObmKj4+PthphBVv06f26tXLYyw1NTXQ6bjUrl3bNn7VVVd5jG3dutV27L59+2zjP/zwg8fYxTo9ak5OjuLi4oKdhk+oB1aXXXaZbdzbe/rUqVMeYz/++KPt2ISEBI+xhg0b2o5t3bq1bfzuu+/2GPNWS44dO2Yb9/RQL0n68ssvbcdGqnCrB+FcCyZMmOAxNnHiRNux+fn5HmMfffSR7dhGjRrZxps1a2Yb95e3z+fMzEzb+Jw5czzG7M4H/OOtFnBFAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABgwXMUEJL69etnG3/33Xdt4++8847H2KBBg/zKKdyF27zp0sVbDxwOh8dYiJXsEitXrpxt3O75ECtXrrQdW7duXdt4586dPcbWr19vOzZShVs9COdaYPf/dv369W3H5uXleYx9//33tmO9/fvaPWehVatWtmPvvPNOjzFvz3LJzc21jXfr1s1jzNszluA7nqMAAAAAwGc0CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABgQaMAAAAAwIJGAQAAAICF/cTWQJAsWrTINu5tLvmLdW50RIZwfVaCnXPnztnG9+zZ4zFWWFhoOzYrK8s2/uOPP9rGgdKwe+6JJO3fv9+vWGkdOnTINv7NN994jC1dutR27Lx58/we27RpU9t4jRo1bOO4sLiiAAAAAMCCRgEAAACABY0CAAAAAAsaBQAAAAAWNAoAAAAALGgUAAAAAFgwPSqCpk+fPh5j3qaHXLdunW18wYIF/qQEIEjs3rP169e3Hfvee+/Zxk+cOOFXTkBJROJ0xt6mJP7uu+88xgoKCmzHnjp1yjb+ww8/2MZxYfl0RSEzM1Nt2rRRlSpVVLNmTfXv31+7du1y2+bMmTPKyMhQtWrVVLlyZQ0YMECHDx8OaNIAgo96AECiFgCRzKdGYf369crIyNCWLVu0atUqnT17Vt26dVNeXp5rm1GjRumDDz7QwoULtX79eh06dEg33nhjwBMHEFzUAwAStQCIZD7derR8+XK3n2fNmqWaNWtq27Ztuvbaa5WTk6M33nhD8+fPV5cuXSRJM2fOVNOmTbVlyxZdddVVln3m5+crPz/f9XNubq4/rwPABUY9ACBRC4BIVqovM+fk5EiSqlatKknatm2bzp49q/T0dNc2TZo0UZ06dbR58+Zi95GZman4+HjXkpKSUpqUAAQJ9QCARC0AIonfjUJhYaFGjhypDh06qEWLFpKkrKwsxcTEKCEhwW3bxMREZWVlFbufcePGKScnx7UcPHjQ35QABAn1AIBELQAijd+zHmVkZGjHjh3auHFjqRJwOp1yOp2l2geA4KIeAJCoBUCk8euKwvDhw7V06VKtXbtWtWvXdq1PSkpSQUGBsrOz3bY/fPiwkpKSSpUogNBEPQAgUQuASOTTFQVjjEaMGKFFixZp3bp1qlu3rls8LS1N5cuX1+rVqzVgwABJ0q5du/T999+rffv2gcsaAfPggw/axp988kmPsUaNGtmOvffee23jd955p8fYzJkzbcd6y/v48eO2cZQe9SA4oqLs/75Tr14923jDhg09xj766CO/cpKkq6++2jY+atQo2/jAgQP9PvbcuXNt4zxHoWxRCy4+r7/+usdYq1atbMeuXLnSNu7pdjQEh0+NQkZGhubPn68lS5aoSpUqrn/M+Ph4VahQQfHx8brzzjs1evRoVa1aVXFxcRoxYoTat29f7KwGAMIX9QCARC0AIplPjcKMGTMkSZ07d3ZbP3PmTA0ZMkSS9OyzzyoqKkoDBgxQfn6+unfvrpdeeikgyQIIHdQDABK1AIhkPt965E1sbKymT5+u6dOn+50UgNBHPQAgUQuASFaq5ygAAAAAiEw0CgAAAAAsaBQAAAAAWDhMSW4uvIByc3MVHx8f7DQiit00pjt37vR7v59++qltvEGDBrbxqlWr+n3scuX8flbgRSsnJ0dxcXHBTsMn1AOrESNG2Mbvuece23j16tU9xubPn2879pJLLvEY69ixo+3Y+vXr28Z//vlnj7ExY8bYjn3zzTdt42fOnLGNX4zCrR5Eai1ITU21jTdt2tRjbPny5aU6duvWrT3GvE1nfPvtt/t9XG9j582b5zFWWFjo93FRPG+1gCsKAAAAACxoFAAAAABY0CgAAAAAsKBRAAAAAGBBowAAAADAgkYBAAAAgAWNAgAAAAALJqO/CEyZMsVjbNmyZbZje/fu7THWrl0727HeHtGxe/dujzG7nIGLmbdnAnibZ7xmzZoeYyNHjvQnpRL56KOPbOOPP/64x9jGjRsDnQ5wwdxxxx0eY/fdd5/t2OTkZI8xu+cNSJLT6bSNd+7c2WPM7vkNkn0dGjt2rO3Yd9991zbOsxJCC1cUAAAAAFjQKAAAAACwoFEAAAAAYEGjAAAAAMCCRgEAAACABY0CAAAAAAsaBQAAAAAWPEfhImD3rIQ5c+ZcwEwAlNaCBQts4xs2bLCNp6SkeIzFxsbajrV7Nsrp06dtx+7atcs2fujQIds4EK7OnTvnMRYdHW07tkaNGh5jZfncE2915NFHH/UYW7VqVaDTQRBxRQEAAACABY0CAAAAAAsaBQAAAAAWNAoAAAAALGgUAAAAAFjQKAAAAACwcBi7+e6CIDc3V/Hx8cFOA4g4OTk5iouLC3YaPqEeAGUj3OpBONcCu/OcnJxsOzY1NdVjrGLFin7nJElnzpzxGNu5c6ft2AMHDpTq2Agd3mqBT1cUMjMz1aZNG1WpUkU1a9ZU//79LXNjd+7cWQ6Hw2255557/MseQMiiHgCQqAVAJPOpUVi/fr0yMjK0ZcsWrVq1SmfPnlW3bt2Ul5fntt1dd92ln376ybVMnTo1oEkDCD7qAQCJWgBEMp+ezLx8+XK3n2fNmqWaNWtq27Ztuvbaa13rK1asqKSkpBLtMz8/X/n5+a6fc3NzfUkJQJBQDwBI1AIgkpXqy8w5OTmSpKpVq7qtnzdvnqpXr64WLVpo3LhxOn36tMd9ZGZmKj4+3rWkpKSUJiUAQUI9ACBRC4BI4veXmQsLC/XHP/5R2dnZ2rhxo2v9q6++qtTUVCUnJ+vLL7/Ugw8+qLZt2+q9994rdj/F/dWAggAEXll+eZF6AISXsqoH1AIrvsyMUOatFvh069FvZWRkaMeOHW6FQJLuvvtu13+3bNlStWrVUteuXbV3717Vr1/fsh+n0ymn0+lvGgBCAPUAgEQtACKNX7ceDR8+XEuXLtXatWtVu3Zt223btWsnSdqzZ48/hwIQ4qgHACRqARCJfLqiYIzRiBEjtGjRIq1bt05169b1Omb79u2SpFq1avmVIIDQRD0AIFELvLH7Ira3L2l/8803gU4H8IlPjUJGRobmz5+vJUuWqEqVKsrKypIkxcfHq0KFCtq7d6/mz5+vXr16qVq1avryyy81atQoXXvttbr88svL5AUACA7qAQCJWgBENOMDScUuM2fONMYY8/3335trr73WVK1a1TidTtOgQQMzduxYk5OTU+Jj5OTkeDwOCwuL/4sv70PqAQtLZC+BrAeejkEtYGEJ/cXb+9DvWY/KSjg/ph0IZWU561FZoR4AZSPc6gG1ACgb3mpBqZ6jAAAAACAy0SgAAAAAsKBRAAAAAGBBowAAAADAgkYBAAAAgAWNAgAAAAALGgUAAAAAFjQKAAAAACxoFAAAAABY0CgAAAAAsAi5RsEYE+wUgIgUju+tcMwZCAfh9t4Kt3yBcOHtvRVyjcLJkyeDnQIQkcLxvRWOOQPhINzeW+GWLxAuvL23HCbE2vTCwkIdOnRIVapUkcPhUG5urlJSUnTw4EHFxcUFO72Qx/nyzcVwvowxOnnypJKTkxUVFXJ/G7BFPSgdzpdvLobzFa71gFpQOpwv31wM56uktaDcBcypRKKiolS7dm3L+ri4uIj9xyoLnC/fRPr5io+PD3YKfqEeBAbnyzeRfr7CsR5QCwKD8+WbSD9fJakF4fPnBAAAAAAXDI0CAAAAAIuQbxScTqcmTJggp9MZ7FTCAufLN5yv8MK/l284X77hfIUP/q18w/nyDefrv0Luy8wAAAAAgi/krygAAAAAuPBoFAAAAABY0CgAAAAAsKBRAAAAAGBBowAAAADAIuQbhenTp+uyyy5TbGys2rVrp08//TTYKYWEDRs2qG/fvkpOTpbD4dDixYvd4sYYjR8/XrVq1VKFChWUnp6ub7/9NjjJBllmZqbatGmjKlWqqGbNmurfv7927drlts2ZM2eUkZGhatWqqXLlyhowYIAOHz4cpIxRHGpB8agFvqEeRAbqQfGoByVHLSiZkG4U3n77bY0ePVoTJkzQ559/rlatWql79+46cuRIsFMLury8PLVq1UrTp08vNj516lS98MILevnll7V161ZVqlRJ3bt315kzZy5wpsG3fv16ZWRkaMuWLVq1apXOnj2rbt26KS8vz7XNqFGj9MEHH2jhwoVav369Dh06pBtvvDGIWeO3qAWeUQt8Qz0If9QDz6gHJUctKCETwtq2bWsyMjJcP58/f94kJyebzMzMIGYVeiSZRYsWuX4uLCw0SUlJ5qmnnnKty87ONk6n07z55ptByDC0HDlyxEgy69evN8b8em7Kly9vFi5c6Nrm66+/NpLM5s2bg5UmfoNaUDLUAt9RD8IP9aBkqAe+oRYUL2SvKBQUFGjbtm1KT093rYuKilJ6ero2b94cxMxC3759+5SVleV27uLj49WuXTvOnaScnBxJUtWqVSVJ27Zt09mzZ93OV5MmTVSnTh3OVwigFviPWuAd9SC8UA/8Rz2wRy0oXsg2CseOHdP58+eVmJjotj4xMVFZWVlByio8FJ0fzp1VYWGhRo4cqQ4dOqhFixaSfj1fMTExSkhIcNuW8xUaqAX+oxbYox6EH+qB/6gHnlELPCsX7ASACykjI0M7duzQxo0bg50KgCCjHgCQqAV2QvaKQvXq1RUdHW35dvnhw4eVlJQUpKzCQ9H54dy5Gz58uJYuXaq1a9eqdu3arvVJSUkqKChQdna22/YX+/kKFdQC/1ELPKMehCfqgf+oB8WjFtgL2UYhJiZGaWlpWr16tWtdYWGhVq9erfbt2wcxs9BXt25dJSUluZ273Nxcbd269aI8d8YYDR8+XIsWLdKaNWtUt25dt3haWprKly/vdr527dql77///qI8X6GGWuA/aoEV9SC8UQ/8Rz1wRy0ooSB/mdrWW2+9ZZxOp5k1a5bZuXOnufvuu01CQoLJysoKdmpBd/LkSfPFF1+YL774wkgyzzzzjPniiy/MgQMHjDHGPPHEEyYhIcEsWbLEfPnll6Zfv36mbt265pdffgly5hfesGHDTHx8vFm3bp356aefXMvp06dd29xzzz2mTp06Zs2aNeazzz4z7du3N+3btw9i1vgtaoFn1ALfUA/CH/XAM+pByVELSiakGwVjjHnxxRdNnTp1TExMjGnbtq3ZsmVLsFMKCWvXrjWSLMvgwYONMb9Og/a3v/3NJCYmGqfTabp27Wp27doV3KSDpLjzJMnMnDnTtc0vv/xi7r33XnPJJZeYihUrmhtuuMH89NNPwUsaFtSC4lELfEM9iAzUg+JRD0qOWlAyDmOMKdtrFgAAAADCTch+RwEAAABA8NAoAAAAALCgUQAAAABgQaMAAAAAwIJGAQAAAIAFjQIAAAAACxoFAAAAABY0CgAAAAAsaBQAAAAAWNAoAAAAALCgUQAAAABg8f8BnxgG7hifnW0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 800x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rotate = RandomRotation(20)\n",
    "shift = RandomShift(3)\n",
    "composed = transforms.Compose([RandomRotation(20),\n",
    "                               RandomShift(3)])\n",
    "\n",
    "# Apply each of the above transforms on sample.\n",
    "fig = plt.figure()\n",
    "sample = transforms.ToPILImage()(train_df.iloc[65,1:].values.reshape((28,28)).astype(np.uint8)[:,:,None])\n",
    "for i, tsfrm in enumerate([rotate, shift, composed]):\n",
    "    transformed_sample = tsfrm(sample)\n",
    "\n",
    "    ax = plt.subplot(1, 3, i + 1)\n",
    "    plt.tight_layout()\n",
    "    ax.set_title(type(tsfrm).__name__)\n",
    "    ax.imshow(np.reshape(np.array(list(transformed_sample.getdata())), (-1,28)), cmap='gray')    \n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):    \n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "          \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(32, 64, kernel_size=3, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "          \n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(p = 0.5),\n",
    "            nn.Linear(64 * 7 * 7, 512),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(p = 0.5),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(p = 0.5),\n",
    "            nn.Linear(512, 10),\n",
    "        )\n",
    "          \n",
    "        for m in self.features.children():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                m.weight.data.fill_(1)\n",
    "                m.bias.data.zero_()\n",
    "        \n",
    "        for m in self.classifier.children():\n",
    "            if isinstance(m, nn.Linear):\n",
    "                nn.init.xavier_uniform(m.weight)\n",
    "            elif isinstance(m, nn.BatchNorm1d):\n",
    "                m.weight.data.fill_(1)\n",
    "                m.bias.data.zero_()\n",
    "                \n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        \n",
    "        return x     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:45: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n"
     ]
    }
   ],
   "source": [
    "model = Net()\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.003)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()\n",
    "    criterion = criterion.cuda()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    model.train()\n",
    "    exp_lr_scheduler.step()\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = Variable(data), Variable(target)\n",
    "        \n",
    "        if torch.cuda.is_available():\n",
    "            data = data.cuda()\n",
    "            target = target.cuda()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (batch_idx + 1)% 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n",
    "                100. * (batch_idx + 1) / len(train_loader), \n",
    "                loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(data_loader):\n",
    "    model.eval()\n",
    "    loss = 0\n",
    "    correct = 0\n",
    "    \n",
    "    for data, target in data_loader:\n",
    "        data, target = Variable(data, volatile=True), Variable(target)\n",
    "        if torch.cuda.is_available():\n",
    "            data = data.cuda()\n",
    "            target = target.cuda()\n",
    "        \n",
    "        output = model(data)\n",
    "        \n",
    "        loss += F.cross_entropy(output, target, size_average=False).item()\n",
    "\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "        \n",
    "    loss /= len(data_loader.dataset)\n",
    "        \n",
    "    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.3f}%)\\n'.format(\n",
    "        loss, correct, len(data_loader.dataset),\n",
    "        100. * correct / len(data_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:17: DeprecationWarning: AFFINE is deprecated and will be removed in Pillow 10 (2023-07-01). Use Transform.AFFINE instead.\n",
      "  app.launch_new_instance()\n",
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:17: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [6400/42000 (15%)]\tLoss: 0.021101\n",
      "Train Epoch: 0 [12800/42000 (30%)]\tLoss: 0.264660\n",
      "Train Epoch: 0 [19200/42000 (46%)]\tLoss: 0.044563\n",
      "Train Epoch: 0 [25600/42000 (61%)]\tLoss: 0.037708\n",
      "Train Epoch: 0 [32000/42000 (76%)]\tLoss: 0.061987\n",
      "Train Epoch: 0 [38400/42000 (91%)]\tLoss: 0.147032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:7: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  import sys\n",
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0503, Accuracy: 41345/42000 (98.440%)\n",
      "\n",
      "Train Epoch: 1 [6400/42000 (15%)]\tLoss: 0.030712\n",
      "Train Epoch: 1 [12800/42000 (30%)]\tLoss: 0.026291\n",
      "Train Epoch: 1 [19200/42000 (46%)]\tLoss: 0.052621\n",
      "Train Epoch: 1 [25600/42000 (61%)]\tLoss: 0.061367\n",
      "Train Epoch: 1 [32000/42000 (76%)]\tLoss: 0.056798\n",
      "Train Epoch: 1 [38400/42000 (91%)]\tLoss: 0.077670\n",
      "\n",
      "Average loss: 0.0447, Accuracy: 41416/42000 (98.610%)\n",
      "\n",
      "Train Epoch: 2 [6400/42000 (15%)]\tLoss: 0.041305\n",
      "Train Epoch: 2 [12800/42000 (30%)]\tLoss: 0.110209\n",
      "Train Epoch: 2 [19200/42000 (46%)]\tLoss: 0.081923\n",
      "Train Epoch: 2 [25600/42000 (61%)]\tLoss: 0.072758\n",
      "Train Epoch: 2 [32000/42000 (76%)]\tLoss: 0.161933\n",
      "Train Epoch: 2 [38400/42000 (91%)]\tLoss: 0.051520\n",
      "\n",
      "Average loss: 0.0419, Accuracy: 41463/42000 (98.721%)\n",
      "\n",
      "Train Epoch: 3 [6400/42000 (15%)]\tLoss: 0.133364\n",
      "Train Epoch: 3 [12800/42000 (30%)]\tLoss: 0.027410\n",
      "Train Epoch: 3 [19200/42000 (46%)]\tLoss: 0.112648\n",
      "Train Epoch: 3 [25600/42000 (61%)]\tLoss: 0.188245\n",
      "Train Epoch: 3 [32000/42000 (76%)]\tLoss: 0.120939\n",
      "Train Epoch: 3 [38400/42000 (91%)]\tLoss: 0.015789\n",
      "\n",
      "Average loss: 0.0411, Accuracy: 41441/42000 (98.669%)\n",
      "\n",
      "Train Epoch: 4 [6400/42000 (15%)]\tLoss: 0.017892\n",
      "Train Epoch: 4 [12800/42000 (30%)]\tLoss: 0.205703\n",
      "Train Epoch: 4 [19200/42000 (46%)]\tLoss: 0.171394\n",
      "Train Epoch: 4 [25600/42000 (61%)]\tLoss: 0.053272\n",
      "Train Epoch: 4 [32000/42000 (76%)]\tLoss: 0.114511\n",
      "Train Epoch: 4 [38400/42000 (91%)]\tLoss: 0.121124\n",
      "\n",
      "Average loss: 0.0387, Accuracy: 41500/42000 (98.810%)\n",
      "\n",
      "Train Epoch: 5 [6400/42000 (15%)]\tLoss: 0.057969\n",
      "Train Epoch: 5 [12800/42000 (30%)]\tLoss: 0.035241\n",
      "Train Epoch: 5 [19200/42000 (46%)]\tLoss: 0.348219\n",
      "Train Epoch: 5 [25600/42000 (61%)]\tLoss: 0.028552\n",
      "Train Epoch: 5 [32000/42000 (76%)]\tLoss: 0.149684\n",
      "Train Epoch: 5 [38400/42000 (91%)]\tLoss: 0.016761\n",
      "\n",
      "Average loss: 0.0336, Accuracy: 41546/42000 (98.919%)\n",
      "\n",
      "Train Epoch: 6 [6400/42000 (15%)]\tLoss: 0.037982\n",
      "Train Epoch: 6 [12800/42000 (30%)]\tLoss: 0.018969\n",
      "Train Epoch: 6 [19200/42000 (46%)]\tLoss: 0.006258\n",
      "Train Epoch: 6 [25600/42000 (61%)]\tLoss: 0.173356\n",
      "Train Epoch: 6 [32000/42000 (76%)]\tLoss: 0.022454\n",
      "Train Epoch: 6 [38400/42000 (91%)]\tLoss: 0.016879\n",
      "\n",
      "Average loss: 0.0353, Accuracy: 41541/42000 (98.907%)\n",
      "\n",
      "Train Epoch: 7 [6400/42000 (15%)]\tLoss: 0.020377\n",
      "Train Epoch: 7 [12800/42000 (30%)]\tLoss: 0.007614\n",
      "Train Epoch: 7 [19200/42000 (46%)]\tLoss: 0.125624\n",
      "Train Epoch: 7 [25600/42000 (61%)]\tLoss: 0.027006\n",
      "Train Epoch: 7 [32000/42000 (76%)]\tLoss: 0.045833\n",
      "Train Epoch: 7 [38400/42000 (91%)]\tLoss: 0.011093\n",
      "\n",
      "Average loss: 0.0316, Accuracy: 41576/42000 (98.990%)\n",
      "\n",
      "Train Epoch: 8 [6400/42000 (15%)]\tLoss: 0.124522\n",
      "Train Epoch: 8 [12800/42000 (30%)]\tLoss: 0.014996\n",
      "Train Epoch: 8 [19200/42000 (46%)]\tLoss: 0.068740\n",
      "Train Epoch: 8 [25600/42000 (61%)]\tLoss: 0.108196\n",
      "Train Epoch: 8 [32000/42000 (76%)]\tLoss: 0.129139\n",
      "Train Epoch: 8 [38400/42000 (91%)]\tLoss: 0.034657\n",
      "\n",
      "Average loss: 0.0300, Accuracy: 41598/42000 (99.043%)\n",
      "\n",
      "Train Epoch: 9 [6400/42000 (15%)]\tLoss: 0.235154\n",
      "Train Epoch: 9 [12800/42000 (30%)]\tLoss: 0.042510\n",
      "Train Epoch: 9 [19200/42000 (46%)]\tLoss: 0.017747\n",
      "Train Epoch: 9 [25600/42000 (61%)]\tLoss: 0.080779\n",
      "Train Epoch: 9 [32000/42000 (76%)]\tLoss: 0.049923\n",
      "Train Epoch: 9 [38400/42000 (91%)]\tLoss: 0.047046\n",
      "\n",
      "Average loss: 0.0297, Accuracy: 41596/42000 (99.038%)\n",
      "\n",
      "Train Epoch: 10 [6400/42000 (15%)]\tLoss: 0.024767\n",
      "Train Epoch: 10 [12800/42000 (30%)]\tLoss: 0.010370\n",
      "Train Epoch: 10 [19200/42000 (46%)]\tLoss: 0.029392\n",
      "Train Epoch: 10 [25600/42000 (61%)]\tLoss: 0.040632\n",
      "Train Epoch: 10 [32000/42000 (76%)]\tLoss: 0.055433\n",
      "Train Epoch: 10 [38400/42000 (91%)]\tLoss: 0.018561\n",
      "\n",
      "Average loss: 0.0292, Accuracy: 41600/42000 (99.048%)\n",
      "\n",
      "Train Epoch: 11 [6400/42000 (15%)]\tLoss: 0.043777\n",
      "Train Epoch: 11 [12800/42000 (30%)]\tLoss: 0.028315\n",
      "Train Epoch: 11 [19200/42000 (46%)]\tLoss: 0.007409\n",
      "Train Epoch: 11 [25600/42000 (61%)]\tLoss: 0.070271\n",
      "Train Epoch: 11 [32000/42000 (76%)]\tLoss: 0.037767\n",
      "Train Epoch: 11 [38400/42000 (91%)]\tLoss: 0.104955\n",
      "\n",
      "Average loss: 0.0286, Accuracy: 41629/42000 (99.117%)\n",
      "\n",
      "Train Epoch: 12 [6400/42000 (15%)]\tLoss: 0.031190\n",
      "Train Epoch: 12 [12800/42000 (30%)]\tLoss: 0.019601\n",
      "Train Epoch: 12 [19200/42000 (46%)]\tLoss: 0.150974\n",
      "Train Epoch: 12 [25600/42000 (61%)]\tLoss: 0.191418\n",
      "Train Epoch: 12 [32000/42000 (76%)]\tLoss: 0.025131\n",
      "Train Epoch: 12 [38400/42000 (91%)]\tLoss: 0.042374\n",
      "\n",
      "Average loss: 0.0277, Accuracy: 41639/42000 (99.140%)\n",
      "\n",
      "Train Epoch: 13 [6400/42000 (15%)]\tLoss: 0.133319\n",
      "Train Epoch: 13 [12800/42000 (30%)]\tLoss: 0.096291\n",
      "Train Epoch: 13 [19200/42000 (46%)]\tLoss: 0.074149\n",
      "Train Epoch: 13 [25600/42000 (61%)]\tLoss: 0.010601\n",
      "Train Epoch: 13 [32000/42000 (76%)]\tLoss: 0.037854\n",
      "Train Epoch: 13 [38400/42000 (91%)]\tLoss: 0.083619\n",
      "\n",
      "Average loss: 0.0286, Accuracy: 41623/42000 (99.102%)\n",
      "\n",
      "Train Epoch: 14 [6400/42000 (15%)]\tLoss: 0.092076\n",
      "Train Epoch: 14 [12800/42000 (30%)]\tLoss: 0.054985\n",
      "Train Epoch: 14 [19200/42000 (46%)]\tLoss: 0.129021\n",
      "Train Epoch: 14 [25600/42000 (61%)]\tLoss: 0.068157\n",
      "Train Epoch: 14 [32000/42000 (76%)]\tLoss: 0.012511\n",
      "Train Epoch: 14 [38400/42000 (91%)]\tLoss: 0.027638\n",
      "\n",
      "Average loss: 0.0278, Accuracy: 41657/42000 (99.183%)\n",
      "\n",
      "Train Epoch: 15 [6400/42000 (15%)]\tLoss: 0.026832\n",
      "Train Epoch: 15 [12800/42000 (30%)]\tLoss: 0.086989\n",
      "Train Epoch: 15 [19200/42000 (46%)]\tLoss: 0.121050\n",
      "Train Epoch: 15 [25600/42000 (61%)]\tLoss: 0.064934\n",
      "Train Epoch: 15 [32000/42000 (76%)]\tLoss: 0.064471\n",
      "Train Epoch: 15 [38400/42000 (91%)]\tLoss: 0.004611\n",
      "\n",
      "Average loss: 0.0288, Accuracy: 41625/42000 (99.107%)\n",
      "\n",
      "Train Epoch: 16 [6400/42000 (15%)]\tLoss: 0.043059\n",
      "Train Epoch: 16 [12800/42000 (30%)]\tLoss: 0.084291\n",
      "Train Epoch: 16 [19200/42000 (46%)]\tLoss: 0.071846\n",
      "Train Epoch: 16 [25600/42000 (61%)]\tLoss: 0.033261\n",
      "Train Epoch: 16 [32000/42000 (76%)]\tLoss: 0.016873\n",
      "Train Epoch: 16 [38400/42000 (91%)]\tLoss: 0.074744\n",
      "\n",
      "Average loss: 0.0278, Accuracy: 41638/42000 (99.138%)\n",
      "\n",
      "Train Epoch: 17 [6400/42000 (15%)]\tLoss: 0.050166\n",
      "Train Epoch: 17 [12800/42000 (30%)]\tLoss: 0.095008\n",
      "Train Epoch: 17 [19200/42000 (46%)]\tLoss: 0.028950\n",
      "Train Epoch: 17 [25600/42000 (61%)]\tLoss: 0.106370\n",
      "Train Epoch: 17 [32000/42000 (76%)]\tLoss: 0.089463\n",
      "Train Epoch: 17 [38400/42000 (91%)]\tLoss: 0.091544\n",
      "\n",
      "Average loss: 0.0283, Accuracy: 41623/42000 (99.102%)\n",
      "\n",
      "Train Epoch: 18 [6400/42000 (15%)]\tLoss: 0.140509\n",
      "Train Epoch: 18 [12800/42000 (30%)]\tLoss: 0.088715\n",
      "Train Epoch: 18 [19200/42000 (46%)]\tLoss: 0.112529\n",
      "Train Epoch: 18 [25600/42000 (61%)]\tLoss: 0.068123\n",
      "Train Epoch: 18 [32000/42000 (76%)]\tLoss: 0.059322\n",
      "Train Epoch: 18 [38400/42000 (91%)]\tLoss: 0.092896\n",
      "\n",
      "Average loss: 0.0277, Accuracy: 41643/42000 (99.150%)\n",
      "\n",
      "Train Epoch: 19 [6400/42000 (15%)]\tLoss: 0.047857\n",
      "Train Epoch: 19 [12800/42000 (30%)]\tLoss: 0.037195\n",
      "Train Epoch: 19 [19200/42000 (46%)]\tLoss: 0.010636\n",
      "Train Epoch: 19 [25600/42000 (61%)]\tLoss: 0.036045\n",
      "Train Epoch: 19 [32000/42000 (76%)]\tLoss: 0.030725\n",
      "Train Epoch: 19 [38400/42000 (91%)]\tLoss: 0.062544\n",
      "\n",
      "Average loss: 0.0282, Accuracy: 41631/42000 (99.121%)\n",
      "\n",
      "Train Epoch: 20 [6400/42000 (15%)]\tLoss: 0.004889\n",
      "Train Epoch: 20 [12800/42000 (30%)]\tLoss: 0.018944\n",
      "Train Epoch: 20 [19200/42000 (46%)]\tLoss: 0.118956\n",
      "Train Epoch: 20 [25600/42000 (61%)]\tLoss: 0.085119\n",
      "Train Epoch: 20 [32000/42000 (76%)]\tLoss: 0.031065\n",
      "Train Epoch: 20 [38400/42000 (91%)]\tLoss: 0.046176\n",
      "\n",
      "Average loss: 0.0274, Accuracy: 41630/42000 (99.119%)\n",
      "\n",
      "Train Epoch: 21 [6400/42000 (15%)]\tLoss: 0.025201\n",
      "Train Epoch: 21 [12800/42000 (30%)]\tLoss: 0.022661\n",
      "Train Epoch: 21 [19200/42000 (46%)]\tLoss: 0.127165\n",
      "Train Epoch: 21 [25600/42000 (61%)]\tLoss: 0.021248\n",
      "Train Epoch: 21 [32000/42000 (76%)]\tLoss: 0.017736\n",
      "Train Epoch: 21 [38400/42000 (91%)]\tLoss: 0.027976\n",
      "\n",
      "Average loss: 0.0293, Accuracy: 41610/42000 (99.071%)\n",
      "\n",
      "Train Epoch: 22 [6400/42000 (15%)]\tLoss: 0.074254\n",
      "Train Epoch: 22 [12800/42000 (30%)]\tLoss: 0.083213\n",
      "Train Epoch: 22 [19200/42000 (46%)]\tLoss: 0.081690\n",
      "Train Epoch: 22 [25600/42000 (61%)]\tLoss: 0.064549\n",
      "Train Epoch: 22 [32000/42000 (76%)]\tLoss: 0.140967\n",
      "Train Epoch: 22 [38400/42000 (91%)]\tLoss: 0.021071\n",
      "\n",
      "Average loss: 0.0283, Accuracy: 41616/42000 (99.086%)\n",
      "\n",
      "Train Epoch: 23 [6400/42000 (15%)]\tLoss: 0.033204\n",
      "Train Epoch: 23 [12800/42000 (30%)]\tLoss: 0.039528\n",
      "Train Epoch: 23 [19200/42000 (46%)]\tLoss: 0.048906\n",
      "Train Epoch: 23 [25600/42000 (61%)]\tLoss: 0.220016\n",
      "Train Epoch: 23 [32000/42000 (76%)]\tLoss: 0.175559\n",
      "Train Epoch: 23 [38400/42000 (91%)]\tLoss: 0.122453\n",
      "\n",
      "Average loss: 0.0272, Accuracy: 41646/42000 (99.157%)\n",
      "\n",
      "Train Epoch: 24 [6400/42000 (15%)]\tLoss: 0.057264\n",
      "Train Epoch: 24 [12800/42000 (30%)]\tLoss: 0.010775\n",
      "Train Epoch: 24 [19200/42000 (46%)]\tLoss: 0.068809\n",
      "Train Epoch: 24 [25600/42000 (61%)]\tLoss: 0.023128\n",
      "Train Epoch: 24 [32000/42000 (76%)]\tLoss: 0.048332\n",
      "Train Epoch: 24 [38400/42000 (91%)]\tLoss: 0.067465\n",
      "\n",
      "Average loss: 0.0292, Accuracy: 41626/42000 (99.110%)\n",
      "\n",
      "Train Epoch: 25 [6400/42000 (15%)]\tLoss: 0.037461\n",
      "Train Epoch: 25 [12800/42000 (30%)]\tLoss: 0.015395\n",
      "Train Epoch: 25 [19200/42000 (46%)]\tLoss: 0.007031\n",
      "Train Epoch: 25 [25600/42000 (61%)]\tLoss: 0.078839\n",
      "Train Epoch: 25 [32000/42000 (76%)]\tLoss: 0.031429\n",
      "Train Epoch: 25 [38400/42000 (91%)]\tLoss: 0.048960\n",
      "\n",
      "Average loss: 0.0270, Accuracy: 41655/42000 (99.179%)\n",
      "\n",
      "Train Epoch: 26 [6400/42000 (15%)]\tLoss: 0.076960\n",
      "Train Epoch: 26 [12800/42000 (30%)]\tLoss: 0.007617\n",
      "Train Epoch: 26 [19200/42000 (46%)]\tLoss: 0.011606\n",
      "Train Epoch: 26 [25600/42000 (61%)]\tLoss: 0.018549\n",
      "Train Epoch: 26 [32000/42000 (76%)]\tLoss: 0.060743\n",
      "Train Epoch: 26 [38400/42000 (91%)]\tLoss: 0.075617\n",
      "\n",
      "Average loss: 0.0288, Accuracy: 41612/42000 (99.076%)\n",
      "\n",
      "Train Epoch: 27 [6400/42000 (15%)]\tLoss: 0.038096\n",
      "Train Epoch: 27 [12800/42000 (30%)]\tLoss: 0.020579\n",
      "Train Epoch: 27 [19200/42000 (46%)]\tLoss: 0.027612\n",
      "Train Epoch: 27 [25600/42000 (61%)]\tLoss: 0.089376\n",
      "Train Epoch: 27 [32000/42000 (76%)]\tLoss: 0.074086\n",
      "Train Epoch: 27 [38400/42000 (91%)]\tLoss: 0.032644\n",
      "\n",
      "Average loss: 0.0272, Accuracy: 41641/42000 (99.145%)\n",
      "\n",
      "Train Epoch: 28 [6400/42000 (15%)]\tLoss: 0.058920\n",
      "Train Epoch: 28 [12800/42000 (30%)]\tLoss: 0.174407\n",
      "Train Epoch: 28 [19200/42000 (46%)]\tLoss: 0.051705\n",
      "Train Epoch: 28 [25600/42000 (61%)]\tLoss: 0.090982\n",
      "Train Epoch: 28 [32000/42000 (76%)]\tLoss: 0.021295\n",
      "Train Epoch: 28 [38400/42000 (91%)]\tLoss: 0.038456\n",
      "\n",
      "Average loss: 0.0273, Accuracy: 41639/42000 (99.140%)\n",
      "\n",
      "Train Epoch: 29 [6400/42000 (15%)]\tLoss: 0.106013\n",
      "Train Epoch: 29 [12800/42000 (30%)]\tLoss: 0.013659\n",
      "Train Epoch: 29 [19200/42000 (46%)]\tLoss: 0.064443\n",
      "Train Epoch: 29 [25600/42000 (61%)]\tLoss: 0.012816\n",
      "Train Epoch: 29 [32000/42000 (76%)]\tLoss: 0.022083\n",
      "Train Epoch: 29 [38400/42000 (91%)]\tLoss: 0.089367\n",
      "\n",
      "Average loss: 0.0287, Accuracy: 41619/42000 (99.093%)\n",
      "\n",
      "Train Epoch: 30 [6400/42000 (15%)]\tLoss: 0.022843\n",
      "Train Epoch: 30 [12800/42000 (30%)]\tLoss: 0.122574\n",
      "Train Epoch: 30 [19200/42000 (46%)]\tLoss: 0.009307\n",
      "Train Epoch: 30 [25600/42000 (61%)]\tLoss: 0.067633\n",
      "Train Epoch: 30 [32000/42000 (76%)]\tLoss: 0.035128\n",
      "Train Epoch: 30 [38400/42000 (91%)]\tLoss: 0.010288\n",
      "\n",
      "Average loss: 0.0291, Accuracy: 41636/42000 (99.133%)\n",
      "\n",
      "Train Epoch: 31 [6400/42000 (15%)]\tLoss: 0.167118\n",
      "Train Epoch: 31 [12800/42000 (30%)]\tLoss: 0.041095\n",
      "Train Epoch: 31 [19200/42000 (46%)]\tLoss: 0.111120\n",
      "Train Epoch: 31 [25600/42000 (61%)]\tLoss: 0.059289\n",
      "Train Epoch: 31 [32000/42000 (76%)]\tLoss: 0.005902\n",
      "Train Epoch: 31 [38400/42000 (91%)]\tLoss: 0.023855\n",
      "\n",
      "Average loss: 0.0280, Accuracy: 41638/42000 (99.138%)\n",
      "\n",
      "Train Epoch: 32 [6400/42000 (15%)]\tLoss: 0.049556\n",
      "Train Epoch: 32 [12800/42000 (30%)]\tLoss: 0.053854\n",
      "Train Epoch: 32 [19200/42000 (46%)]\tLoss: 0.093268\n",
      "Train Epoch: 32 [25600/42000 (61%)]\tLoss: 0.069591\n",
      "Train Epoch: 32 [32000/42000 (76%)]\tLoss: 0.118314\n",
      "Train Epoch: 32 [38400/42000 (91%)]\tLoss: 0.067238\n",
      "\n",
      "Average loss: 0.0284, Accuracy: 41631/42000 (99.121%)\n",
      "\n",
      "Train Epoch: 33 [6400/42000 (15%)]\tLoss: 0.023876\n",
      "Train Epoch: 33 [12800/42000 (30%)]\tLoss: 0.014083\n",
      "Train Epoch: 33 [19200/42000 (46%)]\tLoss: 0.010367\n",
      "Train Epoch: 33 [25600/42000 (61%)]\tLoss: 0.277861\n",
      "Train Epoch: 33 [32000/42000 (76%)]\tLoss: 0.163155\n",
      "Train Epoch: 33 [38400/42000 (91%)]\tLoss: 0.046968\n",
      "\n",
      "Average loss: 0.0281, Accuracy: 41620/42000 (99.095%)\n",
      "\n",
      "Train Epoch: 34 [6400/42000 (15%)]\tLoss: 0.066852\n",
      "Train Epoch: 34 [12800/42000 (30%)]\tLoss: 0.061455\n",
      "Train Epoch: 34 [19200/42000 (46%)]\tLoss: 0.068675\n",
      "Train Epoch: 34 [25600/42000 (61%)]\tLoss: 0.048373\n",
      "Train Epoch: 34 [32000/42000 (76%)]\tLoss: 0.004132\n",
      "Train Epoch: 34 [38400/42000 (91%)]\tLoss: 0.007481\n",
      "\n",
      "Average loss: 0.0290, Accuracy: 41617/42000 (99.088%)\n",
      "\n",
      "Train Epoch: 35 [6400/42000 (15%)]\tLoss: 0.007370\n",
      "Train Epoch: 35 [12800/42000 (30%)]\tLoss: 0.104633\n",
      "Train Epoch: 35 [19200/42000 (46%)]\tLoss: 0.052242\n",
      "Train Epoch: 35 [25600/42000 (61%)]\tLoss: 0.049746\n",
      "Train Epoch: 35 [32000/42000 (76%)]\tLoss: 0.023410\n",
      "Train Epoch: 35 [38400/42000 (91%)]\tLoss: 0.114413\n",
      "\n",
      "Average loss: 0.0282, Accuracy: 41612/42000 (99.076%)\n",
      "\n",
      "Train Epoch: 36 [6400/42000 (15%)]\tLoss: 0.029625\n",
      "Train Epoch: 36 [12800/42000 (30%)]\tLoss: 0.050327\n",
      "Train Epoch: 36 [19200/42000 (46%)]\tLoss: 0.106734\n",
      "Train Epoch: 36 [25600/42000 (61%)]\tLoss: 0.062678\n",
      "Train Epoch: 36 [32000/42000 (76%)]\tLoss: 0.091800\n",
      "Train Epoch: 36 [38400/42000 (91%)]\tLoss: 0.096887\n",
      "\n",
      "Average loss: 0.0293, Accuracy: 41610/42000 (99.071%)\n",
      "\n",
      "Train Epoch: 37 [6400/42000 (15%)]\tLoss: 0.038051\n",
      "Train Epoch: 37 [12800/42000 (30%)]\tLoss: 0.194889\n",
      "Train Epoch: 37 [19200/42000 (46%)]\tLoss: 0.026729\n",
      "Train Epoch: 37 [25600/42000 (61%)]\tLoss: 0.011289\n",
      "Train Epoch: 37 [32000/42000 (76%)]\tLoss: 0.100835\n",
      "Train Epoch: 37 [38400/42000 (91%)]\tLoss: 0.060950\n",
      "\n",
      "Average loss: 0.0285, Accuracy: 41623/42000 (99.102%)\n",
      "\n",
      "Train Epoch: 38 [6400/42000 (15%)]\tLoss: 0.084753\n",
      "Train Epoch: 38 [12800/42000 (30%)]\tLoss: 0.093848\n",
      "Train Epoch: 38 [19200/42000 (46%)]\tLoss: 0.116186\n",
      "Train Epoch: 38 [25600/42000 (61%)]\tLoss: 0.006589\n",
      "Train Epoch: 38 [32000/42000 (76%)]\tLoss: 0.061946\n",
      "Train Epoch: 38 [38400/42000 (91%)]\tLoss: 0.045903\n",
      "\n",
      "Average loss: 0.0270, Accuracy: 41641/42000 (99.145%)\n",
      "\n",
      "Train Epoch: 39 [6400/42000 (15%)]\tLoss: 0.035082\n",
      "Train Epoch: 39 [12800/42000 (30%)]\tLoss: 0.053345\n",
      "Train Epoch: 39 [19200/42000 (46%)]\tLoss: 0.051900\n",
      "Train Epoch: 39 [25600/42000 (61%)]\tLoss: 0.044550\n",
      "Train Epoch: 39 [32000/42000 (76%)]\tLoss: 0.014886\n",
      "Train Epoch: 39 [38400/42000 (91%)]\tLoss: 0.029871\n",
      "\n",
      "Average loss: 0.0282, Accuracy: 41628/42000 (99.114%)\n",
      "\n",
      "Train Epoch: 40 [6400/42000 (15%)]\tLoss: 0.093263\n",
      "Train Epoch: 40 [12800/42000 (30%)]\tLoss: 0.066924\n",
      "Train Epoch: 40 [19200/42000 (46%)]\tLoss: 0.027215\n",
      "Train Epoch: 40 [25600/42000 (61%)]\tLoss: 0.131752\n",
      "Train Epoch: 40 [32000/42000 (76%)]\tLoss: 0.006635\n",
      "Train Epoch: 40 [38400/42000 (91%)]\tLoss: 0.008728\n",
      "\n",
      "Average loss: 0.0279, Accuracy: 41635/42000 (99.131%)\n",
      "\n",
      "Train Epoch: 41 [6400/42000 (15%)]\tLoss: 0.156395\n",
      "Train Epoch: 41 [12800/42000 (30%)]\tLoss: 0.015581\n",
      "Train Epoch: 41 [19200/42000 (46%)]\tLoss: 0.010172\n",
      "Train Epoch: 41 [25600/42000 (61%)]\tLoss: 0.164503\n",
      "Train Epoch: 41 [32000/42000 (76%)]\tLoss: 0.022604\n",
      "Train Epoch: 41 [38400/42000 (91%)]\tLoss: 0.031348\n",
      "\n",
      "Average loss: 0.0286, Accuracy: 41610/42000 (99.071%)\n",
      "\n",
      "Train Epoch: 42 [6400/42000 (15%)]\tLoss: 0.063984\n",
      "Train Epoch: 42 [12800/42000 (30%)]\tLoss: 0.047632\n",
      "Train Epoch: 42 [19200/42000 (46%)]\tLoss: 0.016663\n",
      "Train Epoch: 42 [25600/42000 (61%)]\tLoss: 0.169957\n",
      "Train Epoch: 42 [32000/42000 (76%)]\tLoss: 0.057952\n",
      "Train Epoch: 42 [38400/42000 (91%)]\tLoss: 0.024662\n",
      "\n",
      "Average loss: 0.0292, Accuracy: 41623/42000 (99.102%)\n",
      "\n",
      "Train Epoch: 43 [6400/42000 (15%)]\tLoss: 0.081967\n",
      "Train Epoch: 43 [12800/42000 (30%)]\tLoss: 0.101155\n",
      "Train Epoch: 43 [19200/42000 (46%)]\tLoss: 0.004922\n",
      "Train Epoch: 43 [25600/42000 (61%)]\tLoss: 0.004376\n",
      "Train Epoch: 43 [32000/42000 (76%)]\tLoss: 0.007040\n",
      "Train Epoch: 43 [38400/42000 (91%)]\tLoss: 0.011972\n",
      "\n",
      "Average loss: 0.0291, Accuracy: 41606/42000 (99.062%)\n",
      "\n",
      "Train Epoch: 44 [6400/42000 (15%)]\tLoss: 0.042361\n",
      "Train Epoch: 44 [12800/42000 (30%)]\tLoss: 0.164375\n",
      "Train Epoch: 44 [19200/42000 (46%)]\tLoss: 0.268632\n",
      "Train Epoch: 44 [25600/42000 (61%)]\tLoss: 0.083272\n",
      "Train Epoch: 44 [32000/42000 (76%)]\tLoss: 0.035689\n",
      "Train Epoch: 44 [38400/42000 (91%)]\tLoss: 0.038734\n",
      "\n",
      "Average loss: 0.0267, Accuracy: 41628/42000 (99.114%)\n",
      "\n",
      "Train Epoch: 45 [6400/42000 (15%)]\tLoss: 0.012401\n",
      "Train Epoch: 45 [12800/42000 (30%)]\tLoss: 0.085922\n",
      "Train Epoch: 45 [19200/42000 (46%)]\tLoss: 0.068924\n",
      "Train Epoch: 45 [25600/42000 (61%)]\tLoss: 0.081781\n",
      "Train Epoch: 45 [32000/42000 (76%)]\tLoss: 0.083717\n",
      "Train Epoch: 45 [38400/42000 (91%)]\tLoss: 0.044584\n",
      "\n",
      "Average loss: 0.0290, Accuracy: 41599/42000 (99.045%)\n",
      "\n",
      "Train Epoch: 46 [6400/42000 (15%)]\tLoss: 0.039650\n",
      "Train Epoch: 46 [12800/42000 (30%)]\tLoss: 0.009014\n",
      "Train Epoch: 46 [19200/42000 (46%)]\tLoss: 0.089480\n",
      "Train Epoch: 46 [25600/42000 (61%)]\tLoss: 0.074625\n",
      "Train Epoch: 46 [32000/42000 (76%)]\tLoss: 0.029601\n",
      "Train Epoch: 46 [38400/42000 (91%)]\tLoss: 0.068031\n",
      "\n",
      "Average loss: 0.0274, Accuracy: 41636/42000 (99.133%)\n",
      "\n",
      "Train Epoch: 47 [6400/42000 (15%)]\tLoss: 0.048178\n",
      "Train Epoch: 47 [12800/42000 (30%)]\tLoss: 0.040914\n",
      "Train Epoch: 47 [19200/42000 (46%)]\tLoss: 0.034510\n",
      "Train Epoch: 47 [25600/42000 (61%)]\tLoss: 0.027902\n",
      "Train Epoch: 47 [32000/42000 (76%)]\tLoss: 0.049214\n",
      "Train Epoch: 47 [38400/42000 (91%)]\tLoss: 0.019994\n",
      "\n",
      "Average loss: 0.0270, Accuracy: 41651/42000 (99.169%)\n",
      "\n",
      "Train Epoch: 48 [6400/42000 (15%)]\tLoss: 0.014736\n",
      "Train Epoch: 48 [12800/42000 (30%)]\tLoss: 0.112598\n",
      "Train Epoch: 48 [19200/42000 (46%)]\tLoss: 0.074767\n",
      "Train Epoch: 48 [25600/42000 (61%)]\tLoss: 0.019111\n",
      "Train Epoch: 48 [32000/42000 (76%)]\tLoss: 0.062905\n",
      "Train Epoch: 48 [38400/42000 (91%)]\tLoss: 0.018423\n",
      "\n",
      "Average loss: 0.0283, Accuracy: 41616/42000 (99.086%)\n",
      "\n",
      "Train Epoch: 49 [6400/42000 (15%)]\tLoss: 0.050322\n",
      "Train Epoch: 49 [12800/42000 (30%)]\tLoss: 0.051767\n",
      "Train Epoch: 49 [19200/42000 (46%)]\tLoss: 0.069450\n",
      "Train Epoch: 49 [25600/42000 (61%)]\tLoss: 0.011367\n",
      "Train Epoch: 49 [32000/42000 (76%)]\tLoss: 0.042609\n",
      "Train Epoch: 49 [38400/42000 (91%)]\tLoss: 0.153906\n",
      "\n",
      "Average loss: 0.0282, Accuracy: 41640/42000 (99.143%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(epoch)\n",
    "    evaluate(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prediciton(data_loader):\n",
    "    model.eval()\n",
    "    test_pred = torch.LongTensor()\n",
    "    \n",
    "    for i, data in enumerate(data_loader):\n",
    "        data = Variable(data, volatile=True)\n",
    "        if torch.cuda.is_available():\n",
    "            data = data.cuda()\n",
    "            \n",
    "        output = model(data)\n",
    "        \n",
    "        pred = output.cpu().data.max(1, keepdim=True)[1]\n",
    "        test_pred = torch.cat((test_pred, pred), dim=0)\n",
    "    return test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy/anaconda3/envs/smartcity/lib/python3.7/site-packages/ipykernel_launcher.py:6: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "test_pred = prediciton(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], \n",
    "                      columns=['ImageId', 'Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ImageId</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ImageId  Label\n",
       "0        1      2\n",
       "1        2      0\n",
       "2        3      9\n",
       "3        4      0\n",
       "4        5      3"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'epoch50.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=Net()\n",
    "model.load_state_dict(torch.load('epoch50.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "1fe52f3ecce771a192cba3b635d497f522b395a9137624fe156b81e6f8827725"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 64-bit ('smartcity': conda)",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": ""
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}